Changeset 26

Timestamp:

Nov 19, 2004, 3:29:26 PM (20 years ago)

Author:

ole

Message:

Disk stats for unix (perlite@GA)

Location:

tools/pytools

Files:

• diskreport.py (modified) (3 diffs)
• rundiskreport.py (added)

tools/pytools/diskreport.py

@@ -6 +6 @@
 """
 
+#Useful Unix commands
+ls_cmd = '-exec ls -la {} \\;'
+filter_cmd = 'gawk \'{printf("%s %s %s\\n", $3x, $5, $9)}\' '
+sortsize_cmd = 'sort -k 2,2 -nr '
+sortname_cmd = 'sort -k 1,1 '
 
 
-def tally(FN, col=4):
+def make_filename(s):
+    """Transform argument string into a suitable filename
+    """
 
-    #Tally the size
-    fid = open(FN)
+    s = s.strip()
+    s = s.replace(' ', '_')
+    #s = s.replace('(', '{')
+    #s = s.replace(')', '}')
+    s = s.replace('(', '')
+    s = s.replace(')', '')
+    s = s.replace('__', '_')
+    s = s.replace('/', '_')                    
+        
+    return s
+
+
+def diskstat(findcommand, outfile, headline):
+
+    import time
+
+    tempfile = '/tmp/diskreport_' + str(time.time())
+
+    #Find files
+    cmd = findcommand
+    cmd += ls_cmd + ' | ' + filter_cmd + ' | ' +\
+           sortsize_cmd + ' | ' + sortname_cmd
+
+    cmd += ' > %s 2>/dev/null' %tempfile #Redirect
+    #print cmd
+    os.system(cmd)
+
+    #sort_and_clean(tempfile)
+
+    #Organise them by name (with totals)
+    fid = open(tempfile)
     lines = fid.readlines()
     fid.close()
 
-    x = 0
-    for line in lines[1:]:
-        fields = line.split()
-        try:
-            x += float(fields[col])
-        except:
-            pass
-        
-    return x    
+    D = {}
+    for line in lines:
+        fields = line.strip().split()
 
-def sort_and_clean(FN):
-    fid = open(FN)
-    lines = fid.readlines()
+        username = fields[0]
+        size = long(fields[1])
+        filename = fields[2]
+        if not D.has_key(username):
+            D[username] = []
+        D[username].append( [filename, size] )
+
+    #Find totals per user
+    grand_total = 0
+    totals = {}
+    for username in D:
+        total = 0
+        for filename, size in D[username]:
+            total += size
+
+        totals[username] = total
+        grand_total += total
+
+
+    #Sort users by diskusage
+    S = [(totals[username], username) for username in totals] #List comprehension    
+    S.sort()
+    S.reverse()
+    users = [x[1] for x in S] #Users by diskusage
+
+
+    #Write final result to disk
+    fid = open(outfile, 'w')
+
+    print
+    txt = headline
+    txt += '(%.2f MB):' %(grand_total/1.0e6)
+    fid.write(txt + '\n')
+    print '--------------------------------------------------'
+    print txt
+    print '--------------------------------------------------'    
+    
+
+    fid.write('\n')
+    for username in users:
+        fid.write('------------------------------------------------------\n')
+        txt = '%s (totalling %.2f MB)' %(username, totals[username]/1.0e6)
+        fid.write(txt + '\n')
+        fid.write('------------------------------------------------------\n')
+        print txt
+
+        for filename, size in D[username]:
+            fid.write('    %s (%.2f MB)\n' %(filename, size/1.0e6))
+        fid.write('\n')    
+
+
     fid.close()
 
-    fid = open(FN, 'w')
 
-    fid.write(lines[0]) #Header
 
-    sizes = []
-    A = []
-    for line in lines[1:]:
-        fields = line.split()
-
-        try:
-            x = float(fields[4])
-        except:
-            pass
-            print fields
-        else:
-            sizes.append(x)
-            A.append( fields )    
-        
-    # Schwartzian transform     
-    S = [ (sizes[i], A[i]) for i in range(len(sizes)) ] #List comprehension    
-    S.sort()
-    B = [x[1] for x in S]
-    
-    print A, B
-    
-    
-    return B
-    
 
 
@@ -65 +117 @@
 if len(sys.argv) > 1:
     dir = sys.argv[1]
-    days = int(sys.argv[2])
-    filesize = int(sys.argv[3]) 
+    #days = int(sys.argv[2])
+    #filesize = int(sys.argv[3]) 
 else:   
     dir = '.'
-    days = 180
-    filesize = 500000  #One half megabyte
-
-tempfile = '/tmp/diskreport_' + str(time.time())
+    
+days = 365
+filesize = 1000000  #One megabyte
 
 
@@ -78 +129 @@
 
 # Find the largest directories
-cmd = 'du -k "%s" | sort -nr > %s' %(dir, tempfile)
-#print cmd
-os.system(cmd)
-
-fid = open(tempfile)
-lines = fid.readlines()
-fid.close()
-
 txt = 'Top 50 largest directories on %s (stated in kilo bytes)' %dir
 os.system('echo "%s" > top50.txt' %txt)
-
-fid = open('top50.txt', 'a')
-for line in lines[:50]:
-    l = line.strip()
-    #print l
-    fid.write(l + '\n')
-fid.close()
-
-
-txt = 'The top 50 largest directories are stored in top50.txt'
-print txt
-
-
-# Find old files
-#
-cmd = 'find %s -atime +%d > %s 2>/dev/null' %(dir, days, tempfile)
-#print cmd
+cmd = 'du -k "%s" | sort -nr >> top50_%s.txt' %(dir, make_filename(dir))
 os.system(cmd)
 
 
-fid = open(tempfile)
-lines = fid.readlines()
-fid.close()
+#################################################################
+# Find old files
+#################################################################
+diskstat('find %s -xdev -type f -atime +%d ' %(dir, days),
+         'oldfiles_%s.txt' %make_filename(dir),
+         'Statistics for files on disk %s that haven\'t been accessed for at least %d days '\
+         %(dir, days))
+
+         
+#################################################################
+# Find large files
+#################################################################
+diskstat('find %s -xdev -type f -size +%dc ' %(dir, filesize),
+         'bigfiles_%s.txt' %make_filename(dir),         
+         'Statistics for files on disk %s that are larger than %.2f MB '\
+         %(dir, filesize/1.0e6))
 
 
-txt = 'The oldest files'
-os.system('echo "%s" > oldfiles.txt' %txt)
-oldfiles = []
-for line in lines:
-    l = line.strip()
-    oldfiles.append(l)
-    #Use python os to get stats
-    #Or use: find <path> -name <name> -type f -atime +30 -exec ls -all {} \;
-    #find $1 -type f  -size +2048 -xdev -exec ls -s {} + | /bin/sort -nr
-    #find / -xdev -size +1024 -exec ls -al {} \; | sort -r -k 5
-    
-    #GREAT ONE
-    #find . -type f -size +10 -printf "%s %u %a %k %h%f \n" | sort -nr
-    
-    print os.stat(l)
-    cmd = 'ls -lua "%s" >> oldfiles.txt 2>/dev/null' %l
-    #print cmd
-    os.system(cmd)
+#################################################################
+# Find old and large files
+#################################################################
+diskstat('find %s -xdev -type f -atime +%d -size +%dc '\
+         %(dir, days, filesize),
+         'oldbigfiles_%s.txt' %make_filename(dir),
+         'Statistics for files on disk %s that haven\'t been accessed for at least %d days and that are larger than %.2f kB '\
+         %(dir, days, filesize/1.0e6))
+
+         
 
 
-sort_and_clean('oldfiles.txt')
-
-#Tally the size
-x = tally('oldfiles.txt', 4)
-
-
-txt = '%d files haven\'t been accessed for at least %d days and take up a total of %.2f MB. ' %(len(oldfiles), days, x/1000000)
-print txt + 'See oldfiles.txt.' 
-
-
-
-# Find big files
-#
-cmd = 'find %s -size +%dc > %s 2>/dev/null' %(dir, filesize, tempfile) 
-#print cmd
-os.system(cmd)
-
-
-fid = open(tempfile)
-lines = fid.readlines()
-fid.close()
-
-txt = 'The biggest files'
-os.system('echo "%s" > bigfiles.txt' %txt)
-bigfiles = []
-for line in lines:
-    l = line.strip()
-    bigfiles.append(l)
-    cmd = 'ls -lua "%s" >> bigfiles.txt 2>/dev/null' %l
-    #print cmd
-    os.system(cmd)      
-fid.close()
-
-
-
-#Tally the size
-x = tally('bigfiles.txt', 4)
-
-sort_and_clean('bigfiles.txt')
-
-txt = '%d files are larger than %.2f MB and take up a total of %.2f MB'\
-      %(len(bigfiles), filesize/1000000.0, x/1000000)
-print txt + '. See bigfiles.txt for details' 
-
-
-
-os.remove(tempfile)
-
-
-#Find intersection
-
-intersection = []
-for f in oldfiles:
-   if f in bigfiles:
-       intersection.append(f)
-
-
-os.system('echo "%s" > intersection.txt' %txt)
-for f in intersection:
-    cmd = 'ls -lua "%s" 2>/dev/null >> intersection.txt' %f
-    os.system(cmd)      
-
-
-x = tally('intersection.txt', 4)
-
-sort_and_clean('intersection.txt')
-
-txt = '%d files are larger than %.2f MB,'\
-      %(len(intersection), filesize/1000000.0)
-txt += ' haven\'t been accessed for at least %d days' %days
-txt += ' and take up a total of %.2f MB' %(x/1000000)
-print txt
-
-
-