From 2f7098a8668cfe01b7767d5f81c51c9292397659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakobus=20Sch=C3=BCrz?= Date: Sat, 15 Feb 2020 04:32:16 +0100 Subject: [PATCH] cleanup code --- workdirfs.py | 284 +++++++++++++++++++++++++++------------------------ 1 file changed, 148 insertions(+), 136 deletions(-) diff --git a/workdirfs.py b/workdirfs.py index cbd3f76..8437839 100755 --- a/workdirfs.py +++ b/workdirfs.py @@ -6,7 +6,7 @@ import os import sys import errno -from datetime import datetime, timedelta +from datetime import datetime, timedelta, date import time import fileinput @@ -14,6 +14,8 @@ import fileinput import argparse import gzip import shutil +import json +import traceback try: from fuse import FUSE, FuseOSError, Operations @@ -28,51 +30,167 @@ except: class WorkdirFS(Operations): def __init__(self, args): self.args = args - self.today = datetime.now() - timedelta(hours=self.args.timeoffset) - self.yesterday = datetime.now() - timedelta(hours=self.args.timeoffset) - self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs') - if os.path.exists(os.path.join(self.confdir, 'yesterpath')): - with open(os.path.join(self.confdir, 'yesterpath'), 'r') as fh: - self.yesterpath = fh.readline().strip() + # init configdir and configlast + self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs') + self.configlast = os.path.join(self.confdir, 'yesterday') + + self._checkdir(self.confdir) + + # init archivbase and xdg_archive + if self.args.archive.startswith("/"): + self.archivpathbase = self.args.archive + self.xdgarchivpathbase = self.args.archive else: - self.yesterpath = os.path.join(self.args.archive, 'workdir', self.yesterday.strftime("%Y-%m-%d")) - if not os.path.isdir(self.confdir): - os.mkdir(self.confdir) - with open(os.path.join(self.confdir, 'yesterpath'), 'w') as fh: - fh.write(self.yesterpath) - print("initial yesterpath is {}".format(self.yesterpath)) + self.archivpathbase = os.path.join(os.environ['HOME'], self.args.archive) + self.xdgarchivpathbase = os.path.join('"$HOME', self.args.archive) + + self._xdg() + self._give_me_today() + self.todaypath = self._checkdir(self._give_me_archivpath()) + self.yesterdaypath = self._checkdir(self._give_me_archivpath(self._give_me_yesterday())) + + print("initial yesterdaypath is {}".format(self.yesterdaypath)) + print("initial todaypath is {}".format(self.todaypath)) # Helpers # ======= - def _full_path(self, partial): + def _xdg(self): + foundarchive=False + foundwork=False + xdguserdirs = os.environ['HOME']+'/.config/user-dirs.dirs' + try: + with fileinput.input(xdguserdirs, inplace=True) as fh: + for line in fh: + if line.startswith('XDG_ARCHIVE_DIR'): + print("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase, end='\n') + foundarchive=True + elif line.startswith('XDG_WORK_DIR'): + print("XDG_WORK_DIR=\"$HOME/" + self.args.mountpoint +'"', end='\n') + foundwork=True + else: + print(line, end='') + except: + print("File not existing, create it: {}".format(xdguserdirs)) + + if not foundarchive: + with open(xdguserdirs, 'a') as fh: + fh.write("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase + '\n') + if not foundwork: + with open(xdguserdirs, 'a') as fh: + fh.write('XDG_WORK_DIR=\"$HOME/' + self.args.mountpoint + '"\n') + + def _give_me_today(self): self.today = datetime.now() - timedelta(hours=self.args.timeoffset) - if self.args.yearlydir: - path = os.path.join(os.environ['HOME'], - self.args.archive,"workdir",self.today.strftime("%Y")) - if self.args.monthlydir: - path = os.path.join(path, self.today.strftime("%m")) + return self.today + + def _give_me_yesterday(self): + if os.path.exists(self.configlast): + with open(self.configlast, 'r') as fh: + self.yesterday = datetime.strptime(fh.readline().strip(), "%Y-%m-%d") else: - path = os.path.join(os.environ['HOME'], self.args.archive, "workdir") + self.yesterday = self.today + self._checkdir(self.confdir) + if not os.path.isdir(self.confdir): + os.mkdir(self.confdir) + with open(self.configlast, 'w') as fh: + fh.write(self.yesterday.date().strftime("%Y-%m-%d")) + return self.yesterday + + def _give_me_archivpath(self, _date=None): + archivpathbase = self.archivpathbase + if _date == None: + _date = self.today + if self.args.yearlydir: + archivpathbase = os.path.join(archivpathbase, _date.strftime("%Y")) + if self.args.monthlydir: + archivpathbase = os.path.join(archivpathbase, _date.strftime("%m")) + if self.args.weeklydir: + archivpathbase = os.path.join(archivpathbase, int(_date.strftime("%W"))+1) + self.archivpath = os.path.join(archivpathbase, _date.strftime("%Y-%m-%d")) + return self.archivpath + + def _checkdir(self, path): + if not os.path.exists(path) and not os.path.isdir(path): + try: + os.makedirs(path, exist_ok=True) + print("Created directory {}".format(path), flush=True) + except Exception as e: + print("[-] Error while check dir and zip files: ", e) + +# if yesterdaypath != None and todaypath != None: +# print("yesterdaypath: {}, todaypath: {}".format(yesterdaypath, todaypath)) +# if todaypath != yesterdaypath: +# _zipfiles(yesterdaypath) +# #Create .yesterdaypath in archive with yesterdaypath inside +# wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs') +# print("New yesterdaypath is {}".format(todaypath)) +# with open(os.path.join(wdconfigdir, 'yesterdaypath'), 'w+') as f: +# f.write(todaypath) + + return path + + + def _full_path(self, partial): + + #self._give_me_today() + path = self._give_me_archivpath() + self._give_me_yesterday() if partial.startswith("/"): partial = partial[1:] - with open(os.path.join(self.confdir, 'yesterpath'), 'w') as fh: - fh.write(self.yesterpath) - path = os.path.join(check_dir( - os.path.join(path, self.today.strftime("%Y-%m-%d")), - self.yesterpath), - partial + path = os.path.join( + self._checkdir( self._give_me_archivpath(self._give_me_today())), + partial ) - if self.today > self.yesterday: + if self.today.date() > self.yesterday.date(): + self._cleanup_dirs() + print("zip yesterday: ", self.yesterdaypath) self.yesterday = self.today + self.yesterdaypath = self.todaypath + with open(self.configlast, 'w') as fh: + fh.write(self.today.strftime("%Y-%m-%d")) + +# print("yesterdaypath: {}, todaypath: {}".format(self.yesterdaypath, self.todaypath)) +# print("yesterday: {}, today: {}".format(self.yesterday.date(), self.today.date())) return path + + def _cleanup_dirs(self): + + print("Cleanup dir", self.yesterdaypath) + zip_fileext=".gz" + zip_compressionlevel=5 + for root, dirs, files in os.walk(self.yesterdaypath, topdown=False): + for d in dirs: + print("cleanup",os.path.join(root, d)) + if not _dir == self.todaypath and not os.listdir(os.path.join(root, d)): + print("Directory is empty -> remove it", + os.path.join(root, d)) + os.remove(os.path.join(root, d)) + for f in files: + print("compress", os.path.join(root, f)) + if zip_fileext not in f: + try: + with open(os.path.join(root, f), 'rb') as f_in: + with gzip.open( + os.path.join(root, f+zip_fileext), + 'wb', + compresslevel=zip_compressionlevel) as f_out: + shutil.copyfileobj(f_in, f_out) + + except Exception as e: + print("Error during zipping file {}".format(os.path.join(root, f)), e) + traceback.print_exc() + else: + os.remove(os.path.join(root, f)) + + # Filesystem methods # ================== @@ -191,77 +309,7 @@ class WorkdirFS(Operations): def fsync(self, path, fdatasync, fh): return self.flush(path, fh) -def cleanup_dirs(root): - today = datetime.now() - timedelta(hours=2) - today = today.strftime("%Y-%m-%d") - - for root, dirs, files in os.walk(root, topdown=False): - for _dir in dirs: - print("cleanup",os.path.join(root, _dir)) - if not _dir == today and not os.listdir(os.path.join(root, _dir)): - print("""Directory is empty -> remove it (not now implemented for - testpurpose)""", - os.path.join(root, _dir)) - - -def check_dir(path, yesterpath=None): - if not os.path.exists(path) and not os.path.isdir(path): - #try: - os.makedirs(path, exist_ok=True) - print("Created directory {}".format(path), flush=True) - print("yesterpath: {} todaypath: {}".format(yesterpath, path)) - if yesterpath != None: - if path != yesterpath: - _zipfiles(yesterpath) - #Create .yesterpath in archive with yesterpath inside - wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs') - #with open(os.path.join(wdconfigdir, 'yesterpath'), 'w+') as f: - # f.write(path) - f = open(os.path.join(wdconfigdir, 'yesterpath'), 'w+') - f.write(path) - f.flush() - f.close() - print("update yesterpath") - - #except Exception as e: - # print("[-] Error while check dir and zip files: ", e) - return path - -def _zipfiles(path): - print("Zip files in yesterdays archivdir {}".format(path)) - zip_fileext=".gz" - #zip_compression=zipfile.ZIP_DEFLATED - zip_compressionlevel=5 - files = [] - # r=root, d=directories, f = files - for r, d, f in os.walk(path): - for file in f: - if zip_fileext not in file: - files.append(os.path.join(r, file)) - - for f in files: - print("file to zip: {} -> {}".format(os.path.basename(f), f+'.bzip')) - try: - with open(f, 'rb') as f_in: - with gzip.open( - f+zip_fileext, - 'wb', - compresslevel=zip_compressionlevel) as f_out: - shutil.copyfileobj(f_in, f_out) - -# with ZipFile( -# f+zip_fileext, -# 'w', -# allowZip64=True, -# compression=zip_compression, -# compresslevel=zip_compressionlevel -# ) as zf: -# zf.write(f, os.path.basename(f)) - except Exception as e: - print("Error during zipping file {}".format(f), e) - else: - os.remove(f) @@ -269,43 +317,6 @@ def _zipfiles(path): def main(args): #FUSE(WorkdirFS(root), mountpoint, nothreads=True, foreground=True) - check_dir(os.path.join(os.environ['HOME'], args.mountpoint)) - if args.archive[:0] == "/": - # Archive-Path is absolute path -> use it as is - check_dir(args.archive) - cleanup_dirs(args.archive) - xdg_entry = '"' + args.archive + '"' - else: - # Archive-path is relative, so use is as relative path from users - # homedir - check_dir(os.path.join(os.environ['HOME'], args.archive)) - cleanup_dirs(os.path.join(os.environ['HOME'], args.archive)) - xdg_entry = '"$HOME/' + args.archive + '"' - # first search if configuration exists for xdg-userdirs - # to use it with alias gowork and goarchive - foundarchive=False - foundwork=False - try: - with fileinput.input(os.environ['HOME']+'/.config/user-dirs.dirs', - inplace=True) as fh: - for line in fh: - if line.startswith('XDG_ARCHIVE_DIR'): - print("XDG_ARCHIVE_DIR=" + xdg_entry, end='\n') - foundarchive=True - elif line.startswith('XDG_WORK_DIR'): - print("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"', end='\n') - foundwork=True - else: - print(line, end='') - except: - print("File not existing, create it: {}".format(os.environ['HOME']+'/.config/user-dirs.dirs')) - if not foundarchive: - with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh: - fh.write("XDG_ARCHIVE_DIR=" + xdg_entry + '\n') - if not foundwork: - with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh: - fh.write('XDG_WORK_DIR=\"$HOME/' + args.mountpoint + '"\n') - # start FUSE filesystem FUSE(WorkdirFS(args), os.path.join(os.environ['HOME'], args.mountpoint), nothreads=True, foreground=True) @@ -313,15 +324,16 @@ if __name__ == '__main__': #main(sys.argv[2], sys.argv[1]) parser = argparse.ArgumentParser() parser.add_argument("-a", "--archive", - default='archive', help="Path to archivedir-base") + default='archive/workdir', help="Path to archivedir-base") parser.add_argument("-m", "--mountpoint", default='Work', help='Path to Workdir') parser.add_argument("-t", "--timeoffset", type=int, default=2, help="""If you're working all day till 3 o'clock in the morning, set it to 4, so next day archive-dir will be created 4 hours after midnight. You have 1h tolerance, if you're working one day a little bit longer""") - parser.add_argument("-y", "--yearlydir", action="store_true") + parser.add_argument("-Y", "--yearlydir", action="store_true") parser.add_argument("-M", "--monthlydir", action="store_true") + parser.add_argument("-W", "--weeklydir", action="store_true") args = parser.parse_args() print(args) #root = os.environ['HOME']+'/archive'