From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from svn.comics.unina.it (unknown [143.225.229.147]) by huchra.bufferbloat.net (Postfix) with ESMTP id 364EE200BCB for ; Thu, 26 May 2011 13:00:06 -0700 (PDT) Received: from www-data by svn.comics.unina.it with local (Exim 4.69) (envelope-from ) id 1QPfo6-0002Cp-Sk for bismark-commits@lists.bufferbloat.net; Thu, 26 May 2011 21:01:18 +0200 To: bismark-commits@lists.bufferbloat.net From: nick@svn.comics.unina.it Message-Id: Date: Thu, 26 May 2011 21:01:18 +0200 Subject: [Bismark-commits] rev 351 - trunk/server/scripts X-BeenThere: bismark-commits@lists.bufferbloat.net X-Mailman-Version: 2.1.13 Precedence: list List-Id: Commit log for the bismark source code List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 26 May 2011 20:00:06 -0000 Author: nick Date: 2011-05-26 21:01:18 +0200 (Thu, 26 May 2011) New Revision: 351 Modified: trunk/server/scripts/bdm trunk/server/scripts/organize-archive.py Log: organization script is now functional, publishing to the web every two hours. currently only running for NB105. Other devices to come. Modified: trunk/server/scripts/bdm =================================================================== --- trunk/server/scripts/bdm 2011-05-26 14:08:30 UTC (rev 350) +++ trunk/server/scripts/bdm 2011-05-26 19:01:18 UTC (rev 351) @@ -346,7 +346,7 @@ /usr/sbin/sendmail -t <<-end From: "Bismark Device Manager" To: "Walter" , "Brian" , "Srikanth" - Subject: Recovery Tunnel Notification ($1) + Subject: Bismark Notification System ($1) Date: $(date -R) Content-type: text/plain @@ -557,8 +557,8 @@ echo -ne $GREEN elif [ $off_time -lt 600 ]; then echo -ne $YELLOW - [ $1 ] && sendmail $id "Device $id went offline" else + [ "$1" -a $off_time -lt 900 ] && sendmail $id "Device $id went offline" echo -ne $RED fi # [ $(mysql -NB -u root -e "SELECT deviceid FROM DEVICES WHERE deviceid='$id'" $MYSQL_DB) ] || printf "#" Modified: trunk/server/scripts/organize-archive.py =================================================================== --- trunk/server/scripts/organize-archive.py 2011-05-26 14:08:30 UTC (rev 350) +++ trunk/server/scripts/organize-archive.py 2011-05-26 19:01:18 UTC (rev 351) @@ -8,11 +8,9 @@ import shutil import tarfile -def clean_data(dir,targetdir): - UKY_DIR = os.environ['HOME'] + '/var/archive/UKY-old' - devices = [] - measurements = [] +def filename_to_dir(file): + types = {'arp.gz':'arp', 'csv':'airodump', 'csv.gz':'airodump', @@ -20,12 +18,47 @@ 'filt.csv':'airodump', 'xml.gz':'active'} + # get device name + match = re.search(r'[0-9A-Za-z]+',file) + if match: + dev_name = match.group() + else: + return '' + + # get date + match = re.search(r'([0-9]{9,10})',file) + if match: + timestr = match.group(1) + date = datetime.date.fromtimestamp(float(timestr)) + datedir = str(date.year) + '/' + str(date.month) + else: + return '' + + # get type + match = re.search(r'.*?[0-9]\.(.*)$',file) + if match: + extension = match.group(1) + typedir = types[extension] + else: + return '' + + + return dev_name + '/' + typedir + '/'+ datedir + + + +def clean_data(dir,targetdir): + + UKY_DIR = os.environ['HOME'] + '/var/archive/UKY-old' + devices = [] + measurements = [] + files = os.listdir(dir) for file in files: if os.path.isdir(dir+file): - print 'skipping ' + file + print 'skipping ' + dir+file continue @@ -35,29 +68,8 @@ shutil.move(dir+file, UKY_DIR) continue - # get list of devices - match = re.search(r'[0-9A-Za-z]+',file) - if match: - dev_name = match.group() - - # list of devices - if dev_name not in devices: - devices.append(dev_name) - - match = re.search(r'([0-9]{10})',file) - if match: - timestr = match.group(1) - date = datetime.date.fromtimestamp(float(timestr)) - datedir = str(date.year) + '/' + str(date.month) - - match = re.search(r'.*?[0-9]\.(.*)$',file) - if match: - extension = match.group(1) - typedir = types[extension] - - # make directories and move files - datadir = targetdir + dev_name + '/' + typedir + '/'+ datedir + datadir = targetdir + filename_to_dir(file) if not os.path.exists(datadir): os.makedirs(datadir) @@ -69,42 +81,58 @@ for dev in devices: print dev -def new_device_files(members,device,dir): +def new_device_files(members,device,pubdir): for tarinfo in members: match = re.search(device,tarinfo.name) - if match: + tfile = pubdir + filename_to_dir(tarinfo.name) + '/' + tarinfo.name + #if match: + # print tfile + if match and not os.path.exists(tfile): yield tarinfo -def unpack_backup(device,dir,outdir): +def unpack_backup(device,dir,tmpdir,pubdir): + # get the tarfiles in the archive files = os.listdir(dir) for file in files: + + # make a directory to unpack the tarfile + try: + os.mkdir(tmpdir) + except OSError: + pass + # look for the active measurement files match = re.search(r'xml.tgz',file) if match: # unpack into the unpack dir print file try: tar = tarfile.open(dir+file,'r:gz') - tar.extractall(outdir,members=new_device_files(tar,device)) + + # unpack all device files that aren't yet published into the unpack dir + tar.extractall(tmpdir,members=new_device_files(tar,device,pubdir)) tar.close() except tarfile.ReadError: print "Warning Read Error" + # publish the data with the correct directory structure + clean_data(tmpdir,pubdir) + + # remove the temporary dir + shutil.rmtree(tmpdir) + if __name__ == '__main__': HOME = os.environ['HOME'] + '/' MEASURE_FILE_DIR = 'var/data/' ARCHIVE_DIR = HOME + MEASURE_FILE_DIR + 'old/' BACKUP_DIR = HOME + 'var/backup/' - UNPACK_DIR = ARCHIVE_DIR + 'unpack/' + UNPACK_DIR = '/data/bismark/unpack/' - PUBLISH_DIR = ARCHIVE_DIR + PUBLISH_DIR = '/data/bismark/public/' - clean_data(ARCHIVE_DIR,PUBLISH_DIR) - # restore directory structure from backup - #unpack_backup('NB105',BACKUP_DIR,UNPACK_DIR) - #clean_data(UNPACK_DIR,PUBLISH_DIR) + unpack_backup('NB105',BACKUP_DIR,UNPACK_DIR,PUBLISH_DIR)