Merge branch 'gzip-postcompress'

workdirfs.py

@@ -6,11 +6,15 @@ import os
 import sys
 import errno
 
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date
 import time
 
 import fileinput
 import argparse
+import gzip
+import shutil
+import json
+import traceback
 
 try:
     from fuse import FUSE, FuseOSError, Operations
@@ -25,28 +29,168 @@ except:
 class WorkdirFS(Operations):
     def __init__(self, args):
         self.args = args
-        self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
+
+        # init configdir and configlast
+        self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
+        self.configlast = os.path.join(self.confdir, 'yesterday')
+
+        self._checkdir(self.confdir)
+
+        # init archivbase and xdg_archive
+        if self.args.archive.startswith("/"):
+            self.archivpathbase = self.args.archive
+            self.xdgarchivpathbase = self.args.archive
+        else:
+            self.archivpathbase = os.path.join(os.environ['HOME'], self.args.archive)
+            self.xdgarchivpathbase = os.path.join('"$HOME', self.args.archive)
+
+        self._xdg()
+        self._give_me_today()
+        self.todaypath = self._checkdir(self._give_me_archivpath())
+        self.yesterdaypath = self._checkdir(self._give_me_archivpath(self._give_me_yesterday()))
+
+        print("initial yesterdaypath is {}".format(self.yesterdaypath))
+        print("initial todaypath is {}".format(self.todaypath))
 
     # Helpers
     # =======
 
-    def _full_path(self, partial):
+    def _xdg(self):
-        self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
+        foundarchive=False
-        if self.args.yearlydir:
+        foundwork=False
-            path = os.path.join(os.environ['HOME'],
+        xdguserdirs = os.environ['HOME']+'/.config/user-dirs.dirs'
-                    self.args.archive,"workdir",self.today.strftime("%Y"))
+        try:
-            if self.args.monthlydir:
+            with fileinput.input(xdguserdirs, inplace=True) as fh:
-                path = os.path.join(path, self.today.strftime("%m"))
+                for line in fh:
+                    if line.startswith('XDG_ARCHIVE_DIR'):
+                        print("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase, end='\n')
+                        foundarchive=True
+                    elif line.startswith('XDG_WORK_DIR'):
+                        print("XDG_WORK_DIR=\"$HOME/" + self.args.mountpoint +'"', end='\n')
+                        foundwork=True
                     else:
-            path = os.path.join(os.environ['HOME'], self.args.archive, "workdir")
+                         print(line, end='')
+        except:
+            print("File not existing, create it: {}".format(xdguserdirs))
+
+        if not foundarchive:
+            with open(xdguserdirs, 'a') as fh:
+                fh.write("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase + '\n')
+        if not foundwork:
+            with open(xdguserdirs, 'a') as fh:
+                fh.write('XDG_WORK_DIR=\"$HOME/' + self.args.mountpoint + '"\n')
+
+    def _give_me_today(self):
+        self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
+        return self.today
+
+    def _give_me_yesterday(self):
+        if os.path.exists(self.configlast):
+            with open(self.configlast, 'r') as fh:
+                self.yesterday = datetime.strptime(fh.readline().strip(), "%Y-%m-%d")
+        else:
+            self.yesterday = self.today
+            self._checkdir(self.confdir)
+            if not os.path.isdir(self.confdir):
+                os.mkdir(self.confdir)
+            with open(self.configlast, 'w') as fh:
+                fh.write(self.yesterday.date().strftime("%Y-%m-%d"))
+        return self.yesterday
+
+    def _give_me_archivpath(self, _date=None):
+        archivpathbase = self.archivpathbase
+        if _date == None:
+            _date = self.today
+        if self.args.yearlydir:
+            archivpathbase = os.path.join(archivpathbase, _date.strftime("%Y"))
+        if self.args.monthlydir:
+            archivpathbase = os.path.join(archivpathbase, _date.strftime("%m"))
+        if self.args.weeklydir:
+            archivpathbase = os.path.join(archivpathbase, int(_date.strftime("%W"))+1)
+        self.archivpath = os.path.join(archivpathbase, _date.strftime("%Y-%m-%d"))
+        return self.archivpath
+
+    def _checkdir(self, path):
+        if not os.path.exists(path) and not os.path.isdir(path):
+            try:
+                os.makedirs(path, exist_ok=True)
+                print("Created directory {}".format(path), flush=True)
+            except Exception as e:
+                print("[-] Error while check dir and zip files: ", e)
+
+#        if yesterdaypath != None and todaypath != None:
+#            print("yesterdaypath: {}, todaypath: {}".format(yesterdaypath, todaypath))
+#            if todaypath != yesterdaypath:
+#                _zipfiles(yesterdaypath)
+#                #Create .yesterdaypath in archive with yesterdaypath inside
+#                wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
+#                print("New yesterdaypath is {}".format(todaypath))
+#                with open(os.path.join(wdconfigdir, 'yesterdaypath'), 'w+') as f:
+#                    f.write(todaypath)
+
+        return path
+
+
+    def _full_path(self, partial):
+
+        #self._give_me_today()
+        path = self._give_me_archivpath()
+        self._give_me_yesterday()
 
         if partial.startswith("/"):
             partial = partial[1:]
 
-        path = os.path.join(check_dir(os.path.join(path, self.today.strftime("%Y-%m-%d"))), partial)
 
+        path = os.path.join(
+                self._checkdir( self._give_me_archivpath(self._give_me_today())),
+                partial
+            )
+
+        if self.today.date() > self.yesterday.date():
+            self._cleanup_dirs()
+            print("zip yesterday: ", self.yesterdaypath)
+            self.yesterday = self.today
+            self.yesterdaypath = self.todaypath
+
+        with open(self.configlast, 'w') as fh:
+            fh.write(self.today.strftime("%Y-%m-%d"))
+
+#        print("yesterdaypath: {}, todaypath: {}".format(self.yesterdaypath, self.todaypath))
+#        print("yesterday: {}, today: {}".format(self.yesterday.date(), self.today.date()))
         return path
 
+
+    def _cleanup_dirs(self):
+
+        print("Cleanup dir", self.yesterdaypath)
+        zip_fileext=".gz"
+        zip_compressionlevel=5
+        for root, dirs, files in os.walk(self.yesterdaypath, topdown=False):
+            for d in dirs:
+                print("cleanup",os.path.join(root, d))
+                if not _dir == self.todaypath and not os.listdir(os.path.join(root, d)):
+                    print("Directory is empty -> remove it",
+                          os.path.join(root, d))
+                    os.remove(os.path.join(root, d))
+            if self.args.compress:
+                for f in files:
+                    print("compress", os.path.join(root, f))
+                    if zip_fileext not in f:
+                        try:
+                            with open(os.path.join(root, f), 'rb') as f_in:
+                                with gzip.open(
+                                        os.path.join(root, f+zip_fileext),
+                                        'wb',
+                                        compresslevel=zip_compressionlevel) as f_out:
+                                    shutil.copyfileobj(f_in, f_out)
+
+                        except Exception as e:
+                            print("Error during zipping file {}".format(os.path.join(root, f)), e)
+                            traceback.print_exc()
+                        else:
+                            os.remove(os.path.join(root, f))
+
+
     # Filesystem methods
     # ==================
 
@@ -165,60 +309,14 @@ class WorkdirFS(Operations):
     def fsync(self, path, fdatasync, fh):
         return self.flush(path, fh)
 
-def cleanup_dirs(root):
-
-    today = datetime.now() - timedelta(hours=2)
-    today = today.strftime("%Y-%m-%d")
-
-    for root, dirs, files in os.walk(root, topdown=False):
-        for _dir in dirs:
-            print("cleanup",os.path.join(root, _dir))
-            if not _dir == today and not os.listdir(os.path.join(root, _dir)):
-                print("""Directory is empty -> remove it (not now implemented for
-                        testpurpose)""",
-                      os.path.join(root, _dir))
 
 
-def check_dir(path):
+
-    checkdir = os.path.isdir(path)
+
-    if not checkdir:
+
-        try:
-            os.makedirs(path, exist_ok=True)
-            print("Created directory {}".format(path), flush=True)
-        except:
-            print("[-] Makedir error")
-    return path
 
 def main(args):
     #FUSE(WorkdirFS(root), mountpoint, nothreads=True, foreground=True)
-    check_dir(os.path.join(os.environ['HOME'], args.archive))
-    check_dir(os.path.join(os.environ['HOME'], args.mountpoint))
-    cleanup_dirs(os.path.join(os.environ['HOME'], args.archive))
-    # first search if configuration exists for xdg-userdirs
-    # to use it with alias gowork and goarchive
-    foundarchive=False
-    foundwork=False
-    try:
-        with fileinput.input(os.environ['HOME']+'/.config/user-dirs.dirs',
-                inplace=True) as fh:
-            for line in fh:
-                if line.startswith('XDG_ARCHIVE_DIR'):
-                    print("XDG_ARCHIVE_DIR=\"$HOME/"+args.archive+'"', end='\n')
-                    foundarchive=True
-                elif line.startswith('XDG_WORK_DIR'):
-                    print("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"', end='\n')
-                    foundwork=True
-                else:
-                     print(line, end='')
-    except:
-        print("File not existing, create it: {}".format(os.environ['HOME']+'/.config/user-dirs.dirs'))
-    if not foundarchive:
-        with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
-            fh.write("XDG_ARCHIVE_DIR=\"$HOME/"+args.archive+'"\n')
-    if not foundwork:
-        with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
-            fh.write("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"\n')
-
     # start FUSE filesystem
     FUSE(WorkdirFS(args), os.path.join(os.environ['HOME'], args.mountpoint), nothreads=True, foreground=True)
 
@@ -226,15 +324,31 @@ if __name__ == '__main__':
     #main(sys.argv[2], sys.argv[1])
     parser = argparse.ArgumentParser()
     parser.add_argument("-a", "--archive",
-            default='archive', help="Path to archivedir-base")
+            default='archive/workdir', help="""Path to archivedir-base. When path
+            starts with "/", it's an absolute path, else it is handled as path
+            relative to users home.
+            Defaults to »archive/workdir« """)
     parser.add_argument("-m", "--mountpoint",
-            default='Work', help='Path to Workdir')
+            default='Work', help="""Path to Workdir. This path is always
-    parser.add_argument("-t", "--timeoffset", type=int, default=2, help="""If you're working
+            relative to users homedir. "/" at the begin get removed.
+            Defaults to »Work«""")
+    parser.add_argument("-t", "--timeoffset", type=int, default=4, help="""If you're working
             all day till 3 o'clock in the morning, set it to 4, so next day
             archive-dir will be created 4 hours after midnight. You have 1h
-            tolerance, if you're working one day a little bit longer""")
+            tolerance, if you're working one day a little bit longer.
-    parser.add_argument("-y", "--yearlydir", action="store_true")
+            Defaults to »4«""")
-    parser.add_argument("-M", "--monthlydir", action="store_true")
+    parser.add_argument("-Y", "--yearlydir", action="store_true",
+            help="""Create a yearly directory - named YYYY - under »archive«.
+    Defaults to »False«""")
+    parser.add_argument("-M", "--monthlydir", action="store_true",
+            help="""Create a monthly directory - named MM - under »archive«.
+    Defaults to »False«""")
+    parser.add_argument("-W", "--weeklydir", action="store_true",
+            help="""Create a weekly directory - named WW - under »archive«.
+    Defaults to »False«""")
+    parser.add_argument("-C", "--compress", action="store_false",
+            help="""Compress each file in archive
+            Defaults to »True«""")
     args = parser.parse_args()
     print(args)
     #root = os.environ['HOME']+'/archive'