* [Bismark-commits] rev 351 - trunk/server/scripts
@ 2011-05-26 19:01 nick
0 siblings, 0 replies; only message in thread
From: nick @ 2011-05-26 19:01 UTC (permalink / raw)
To: bismark-commits
Author: nick
Date: 2011-05-26 21:01:18 +0200 (Thu, 26 May 2011)
New Revision: 351
Modified:
trunk/server/scripts/bdm
trunk/server/scripts/organize-archive.py
Log:
organization script is now functional, publishing to the web every two hours.
currently only running for NB105. Other devices to come.
Modified: trunk/server/scripts/bdm
===================================================================
--- trunk/server/scripts/bdm 2011-05-26 14:08:30 UTC (rev 350)
+++ trunk/server/scripts/bdm 2011-05-26 19:01:18 UTC (rev 351)
@@ -346,7 +346,7 @@
/usr/sbin/sendmail -t <<-end
From: "Bismark Device Manager" <do-not-reply@gtnoise.net>
To: "Walter" <walter.dedonato@unina.it>, "Brian" <bpoole@cc.gatech.edu>, "Srikanth" <srikanth@gatech.edu>
- Subject: Recovery Tunnel Notification ($1)
+ Subject: Bismark Notification System ($1)
Date: $(date -R)
Content-type: text/plain
@@ -557,8 +557,8 @@
echo -ne $GREEN
elif [ $off_time -lt 600 ]; then
echo -ne $YELLOW
- [ $1 ] && sendmail $id "Device $id went offline"
else
+ [ "$1" -a $off_time -lt 900 ] && sendmail $id "Device $id went offline"
echo -ne $RED
fi
# [ $(mysql -NB -u root -e "SELECT deviceid FROM DEVICES WHERE deviceid='$id'" $MYSQL_DB) ] || printf "#"
Modified: trunk/server/scripts/organize-archive.py
===================================================================
--- trunk/server/scripts/organize-archive.py 2011-05-26 14:08:30 UTC (rev 350)
+++ trunk/server/scripts/organize-archive.py 2011-05-26 19:01:18 UTC (rev 351)
@@ -8,11 +8,9 @@
import shutil
import tarfile
-def clean_data(dir,targetdir):
- UKY_DIR = os.environ['HOME'] + '/var/archive/UKY-old'
- devices = []
- measurements = []
+def filename_to_dir(file):
+
types = {'arp.gz':'arp',
'csv':'airodump',
'csv.gz':'airodump',
@@ -20,12 +18,47 @@
'filt.csv':'airodump',
'xml.gz':'active'}
+ # get device name
+ match = re.search(r'[0-9A-Za-z]+',file)
+ if match:
+ dev_name = match.group()
+ else:
+ return ''
+
+ # get date
+ match = re.search(r'([0-9]{9,10})',file)
+ if match:
+ timestr = match.group(1)
+ date = datetime.date.fromtimestamp(float(timestr))
+ datedir = str(date.year) + '/' + str(date.month)
+ else:
+ return ''
+
+ # get type
+ match = re.search(r'.*?[0-9]\.(.*)$',file)
+ if match:
+ extension = match.group(1)
+ typedir = types[extension]
+ else:
+ return ''
+
+
+ return dev_name + '/' + typedir + '/'+ datedir
+
+
+
+def clean_data(dir,targetdir):
+
+ UKY_DIR = os.environ['HOME'] + '/var/archive/UKY-old'
+ devices = []
+ measurements = []
+
files = os.listdir(dir)
for file in files:
if os.path.isdir(dir+file):
- print 'skipping ' + file
+ print 'skipping ' + dir+file
continue
@@ -35,29 +68,8 @@
shutil.move(dir+file, UKY_DIR)
continue
- # get list of devices
- match = re.search(r'[0-9A-Za-z]+',file)
- if match:
- dev_name = match.group()
-
- # list of devices
- if dev_name not in devices:
- devices.append(dev_name)
-
- match = re.search(r'([0-9]{10})',file)
- if match:
- timestr = match.group(1)
- date = datetime.date.fromtimestamp(float(timestr))
- datedir = str(date.year) + '/' + str(date.month)
-
- match = re.search(r'.*?[0-9]\.(.*)$',file)
- if match:
- extension = match.group(1)
- typedir = types[extension]
-
-
# make directories and move files
- datadir = targetdir + dev_name + '/' + typedir + '/'+ datedir
+ datadir = targetdir + filename_to_dir(file)
if not os.path.exists(datadir):
os.makedirs(datadir)
@@ -69,42 +81,58 @@
for dev in devices:
print dev
-def new_device_files(members,device,dir):
+def new_device_files(members,device,pubdir):
for tarinfo in members:
match = re.search(device,tarinfo.name)
- if match:
+ tfile = pubdir + filename_to_dir(tarinfo.name) + '/' + tarinfo.name
+ #if match:
+ # print tfile
+ if match and not os.path.exists(tfile):
yield tarinfo
-def unpack_backup(device,dir,outdir):
+def unpack_backup(device,dir,tmpdir,pubdir):
+ # get the tarfiles in the archive
files = os.listdir(dir)
for file in files:
+
+ # make a directory to unpack the tarfile
+ try:
+ os.mkdir(tmpdir)
+ except OSError:
+ pass
+ # look for the active measurement files
match = re.search(r'xml.tgz',file)
if match:
# unpack into the unpack dir
print file
try:
tar = tarfile.open(dir+file,'r:gz')
- tar.extractall(outdir,members=new_device_files(tar,device))
+
+ # unpack all device files that aren't yet published into the unpack dir
+ tar.extractall(tmpdir,members=new_device_files(tar,device,pubdir))
tar.close()
except tarfile.ReadError:
print "Warning Read Error"
+ # publish the data with the correct directory structure
+ clean_data(tmpdir,pubdir)
+
+ # remove the temporary dir
+ shutil.rmtree(tmpdir)
+
if __name__ == '__main__':
HOME = os.environ['HOME'] + '/'
MEASURE_FILE_DIR = 'var/data/'
ARCHIVE_DIR = HOME + MEASURE_FILE_DIR + 'old/'
BACKUP_DIR = HOME + 'var/backup/'
- UNPACK_DIR = ARCHIVE_DIR + 'unpack/'
+ UNPACK_DIR = '/data/bismark/unpack/'
- PUBLISH_DIR = ARCHIVE_DIR
+ PUBLISH_DIR = '/data/bismark/public/'
- clean_data(ARCHIVE_DIR,PUBLISH_DIR)
-
# restore directory structure from backup
- #unpack_backup('NB105',BACKUP_DIR,UNPACK_DIR)
- #clean_data(UNPACK_DIR,PUBLISH_DIR)
+ unpack_backup('NB105',BACKUP_DIR,UNPACK_DIR,PUBLISH_DIR)
^ permalink raw reply [flat|nested] only message in thread
only message in thread, other threads:[~2011-05-26 20:00 UTC | newest]
Thread overview: (only message) (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2011-05-26 19:01 [Bismark-commits] rev 351 - trunk/server/scripts nick
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox