Merge branch 'gzip-postcompress'

This commit is contained in:
Jakobus Schürz 2020-02-15 09:13:09 +01:00
commit 875343e0d7

View file

@ -6,11 +6,15 @@ import os
import sys import sys
import errno import errno
from datetime import datetime, timedelta from datetime import datetime, timedelta, date
import time import time
import fileinput import fileinput
import argparse import argparse
import gzip
import shutil
import json
import traceback
try: try:
from fuse import FUSE, FuseOSError, Operations from fuse import FUSE, FuseOSError, Operations
@ -25,28 +29,168 @@ except:
class WorkdirFS(Operations): class WorkdirFS(Operations):
def __init__(self, args): def __init__(self, args):
self.args = args self.args = args
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
# init configdir and configlast
self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
self.configlast = os.path.join(self.confdir, 'yesterday')
self._checkdir(self.confdir)
# init archivbase and xdg_archive
if self.args.archive.startswith("/"):
self.archivpathbase = self.args.archive
self.xdgarchivpathbase = self.args.archive
else:
self.archivpathbase = os.path.join(os.environ['HOME'], self.args.archive)
self.xdgarchivpathbase = os.path.join('"$HOME', self.args.archive)
self._xdg()
self._give_me_today()
self.todaypath = self._checkdir(self._give_me_archivpath())
self.yesterdaypath = self._checkdir(self._give_me_archivpath(self._give_me_yesterday()))
print("initial yesterdaypath is {}".format(self.yesterdaypath))
print("initial todaypath is {}".format(self.todaypath))
# Helpers # Helpers
# ======= # =======
def _full_path(self, partial): def _xdg(self):
foundarchive=False
foundwork=False
xdguserdirs = os.environ['HOME']+'/.config/user-dirs.dirs'
try:
with fileinput.input(xdguserdirs, inplace=True) as fh:
for line in fh:
if line.startswith('XDG_ARCHIVE_DIR'):
print("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase, end='\n')
foundarchive=True
elif line.startswith('XDG_WORK_DIR'):
print("XDG_WORK_DIR=\"$HOME/" + self.args.mountpoint +'"', end='\n')
foundwork=True
else:
print(line, end='')
except:
print("File not existing, create it: {}".format(xdguserdirs))
if not foundarchive:
with open(xdguserdirs, 'a') as fh:
fh.write("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase + '\n')
if not foundwork:
with open(xdguserdirs, 'a') as fh:
fh.write('XDG_WORK_DIR=\"$HOME/' + self.args.mountpoint + '"\n')
def _give_me_today(self):
self.today = datetime.now() - timedelta(hours=self.args.timeoffset) self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
if self.args.yearlydir: return self.today
path = os.path.join(os.environ['HOME'],
self.args.archive,"workdir",self.today.strftime("%Y")) def _give_me_yesterday(self):
if self.args.monthlydir: if os.path.exists(self.configlast):
path = os.path.join(path, self.today.strftime("%m")) with open(self.configlast, 'r') as fh:
self.yesterday = datetime.strptime(fh.readline().strip(), "%Y-%m-%d")
else: else:
path = os.path.join(os.environ['HOME'], self.args.archive, "workdir") self.yesterday = self.today
self._checkdir(self.confdir)
if not os.path.isdir(self.confdir):
os.mkdir(self.confdir)
with open(self.configlast, 'w') as fh:
fh.write(self.yesterday.date().strftime("%Y-%m-%d"))
return self.yesterday
def _give_me_archivpath(self, _date=None):
archivpathbase = self.archivpathbase
if _date == None:
_date = self.today
if self.args.yearlydir:
archivpathbase = os.path.join(archivpathbase, _date.strftime("%Y"))
if self.args.monthlydir:
archivpathbase = os.path.join(archivpathbase, _date.strftime("%m"))
if self.args.weeklydir:
archivpathbase = os.path.join(archivpathbase, int(_date.strftime("%W"))+1)
self.archivpath = os.path.join(archivpathbase, _date.strftime("%Y-%m-%d"))
return self.archivpath
def _checkdir(self, path):
if not os.path.exists(path) and not os.path.isdir(path):
try:
os.makedirs(path, exist_ok=True)
print("Created directory {}".format(path), flush=True)
except Exception as e:
print("[-] Error while check dir and zip files: ", e)
# if yesterdaypath != None and todaypath != None:
# print("yesterdaypath: {}, todaypath: {}".format(yesterdaypath, todaypath))
# if todaypath != yesterdaypath:
# _zipfiles(yesterdaypath)
# #Create .yesterdaypath in archive with yesterdaypath inside
# wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
# print("New yesterdaypath is {}".format(todaypath))
# with open(os.path.join(wdconfigdir, 'yesterdaypath'), 'w+') as f:
# f.write(todaypath)
return path
def _full_path(self, partial):
#self._give_me_today()
path = self._give_me_archivpath()
self._give_me_yesterday()
if partial.startswith("/"): if partial.startswith("/"):
partial = partial[1:] partial = partial[1:]
path = os.path.join(check_dir(os.path.join(path, self.today.strftime("%Y-%m-%d"))), partial)
path = os.path.join(
self._checkdir( self._give_me_archivpath(self._give_me_today())),
partial
)
if self.today.date() > self.yesterday.date():
self._cleanup_dirs()
print("zip yesterday: ", self.yesterdaypath)
self.yesterday = self.today
self.yesterdaypath = self.todaypath
with open(self.configlast, 'w') as fh:
fh.write(self.today.strftime("%Y-%m-%d"))
# print("yesterdaypath: {}, todaypath: {}".format(self.yesterdaypath, self.todaypath))
# print("yesterday: {}, today: {}".format(self.yesterday.date(), self.today.date()))
return path return path
def _cleanup_dirs(self):
print("Cleanup dir", self.yesterdaypath)
zip_fileext=".gz"
zip_compressionlevel=5
for root, dirs, files in os.walk(self.yesterdaypath, topdown=False):
for d in dirs:
print("cleanup",os.path.join(root, d))
if not _dir == self.todaypath and not os.listdir(os.path.join(root, d)):
print("Directory is empty -> remove it",
os.path.join(root, d))
os.remove(os.path.join(root, d))
if self.args.compress:
for f in files:
print("compress", os.path.join(root, f))
if zip_fileext not in f:
try:
with open(os.path.join(root, f), 'rb') as f_in:
with gzip.open(
os.path.join(root, f+zip_fileext),
'wb',
compresslevel=zip_compressionlevel) as f_out:
shutil.copyfileobj(f_in, f_out)
except Exception as e:
print("Error during zipping file {}".format(os.path.join(root, f)), e)
traceback.print_exc()
else:
os.remove(os.path.join(root, f))
# Filesystem methods # Filesystem methods
# ================== # ==================
@ -165,60 +309,14 @@ class WorkdirFS(Operations):
def fsync(self, path, fdatasync, fh): def fsync(self, path, fdatasync, fh):
return self.flush(path, fh) return self.flush(path, fh)
def cleanup_dirs(root):
today = datetime.now() - timedelta(hours=2)
today = today.strftime("%Y-%m-%d")
for root, dirs, files in os.walk(root, topdown=False):
for _dir in dirs:
print("cleanup",os.path.join(root, _dir))
if not _dir == today and not os.listdir(os.path.join(root, _dir)):
print("""Directory is empty -> remove it (not now implemented for
testpurpose)""",
os.path.join(root, _dir))
def check_dir(path):
checkdir = os.path.isdir(path)
if not checkdir:
try:
os.makedirs(path, exist_ok=True)
print("Created directory {}".format(path), flush=True)
except:
print("[-] Makedir error")
return path
def main(args): def main(args):
#FUSE(WorkdirFS(root), mountpoint, nothreads=True, foreground=True) #FUSE(WorkdirFS(root), mountpoint, nothreads=True, foreground=True)
check_dir(os.path.join(os.environ['HOME'], args.archive))
check_dir(os.path.join(os.environ['HOME'], args.mountpoint))
cleanup_dirs(os.path.join(os.environ['HOME'], args.archive))
# first search if configuration exists for xdg-userdirs
# to use it with alias gowork and goarchive
foundarchive=False
foundwork=False
try:
with fileinput.input(os.environ['HOME']+'/.config/user-dirs.dirs',
inplace=True) as fh:
for line in fh:
if line.startswith('XDG_ARCHIVE_DIR'):
print("XDG_ARCHIVE_DIR=\"$HOME/"+args.archive+'"', end='\n')
foundarchive=True
elif line.startswith('XDG_WORK_DIR'):
print("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"', end='\n')
foundwork=True
else:
print(line, end='')
except:
print("File not existing, create it: {}".format(os.environ['HOME']+'/.config/user-dirs.dirs'))
if not foundarchive:
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
fh.write("XDG_ARCHIVE_DIR=\"$HOME/"+args.archive+'"\n')
if not foundwork:
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
fh.write("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"\n')
# start FUSE filesystem # start FUSE filesystem
FUSE(WorkdirFS(args), os.path.join(os.environ['HOME'], args.mountpoint), nothreads=True, foreground=True) FUSE(WorkdirFS(args), os.path.join(os.environ['HOME'], args.mountpoint), nothreads=True, foreground=True)
@ -226,15 +324,31 @@ if __name__ == '__main__':
#main(sys.argv[2], sys.argv[1]) #main(sys.argv[2], sys.argv[1])
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("-a", "--archive", parser.add_argument("-a", "--archive",
default='archive', help="Path to archivedir-base") default='archive/workdir', help="""Path to archivedir-base. When path
starts with "/", it's an absolute path, else it is handled as path
relative to users home.
Defaults to »archive/workdir« """)
parser.add_argument("-m", "--mountpoint", parser.add_argument("-m", "--mountpoint",
default='Work', help='Path to Workdir') default='Work', help="""Path to Workdir. This path is always
parser.add_argument("-t", "--timeoffset", type=int, default=2, help="""If you're working relative to users homedir. "/" at the begin get removed.
Defaults to »Work«""")
parser.add_argument("-t", "--timeoffset", type=int, default=4, help="""If you're working
all day till 3 o'clock in the morning, set it to 4, so next day all day till 3 o'clock in the morning, set it to 4, so next day
archive-dir will be created 4 hours after midnight. You have 1h archive-dir will be created 4 hours after midnight. You have 1h
tolerance, if you're working one day a little bit longer""") tolerance, if you're working one day a little bit longer.
parser.add_argument("-y", "--yearlydir", action="store_true") Defaults to »4«""")
parser.add_argument("-M", "--monthlydir", action="store_true") parser.add_argument("-Y", "--yearlydir", action="store_true",
help="""Create a yearly directory - named YYYY - under »archive«.
Defaults to »False«""")
parser.add_argument("-M", "--monthlydir", action="store_true",
help="""Create a monthly directory - named MM - under »archive«.
Defaults to »False«""")
parser.add_argument("-W", "--weeklydir", action="store_true",
help="""Create a weekly directory - named WW - under »archive«.
Defaults to »False«""")
parser.add_argument("-C", "--compress", action="store_false",
help="""Compress each file in archive
Defaults to »True«""")
args = parser.parse_args() args = parser.parse_args()
print(args) print(args)
#root = os.environ['HOME']+'/archive' #root = os.environ['HOME']+'/archive'