cleanup code

This commit is contained in:
Jakobus Schürz 2020-02-15 04:32:16 +01:00
parent 73a6f034a5
commit 2f7098a866

View file

@ -6,7 +6,7 @@ import os
import sys
import errno
from datetime import datetime, timedelta
from datetime import datetime, timedelta, date
import time
import fileinput
@ -14,6 +14,8 @@ import fileinput
import argparse
import gzip
import shutil
import json
import traceback
try:
from fuse import FUSE, FuseOSError, Operations
@ -28,51 +30,167 @@ except:
class WorkdirFS(Operations):
def __init__(self, args):
self.args = args
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
self.yesterday = datetime.now() - timedelta(hours=self.args.timeoffset)
self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
if os.path.exists(os.path.join(self.confdir, 'yesterpath')):
with open(os.path.join(self.confdir, 'yesterpath'), 'r') as fh:
self.yesterpath = fh.readline().strip()
# init configdir and configlast
self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
self.configlast = os.path.join(self.confdir, 'yesterday')
self._checkdir(self.confdir)
# init archivbase and xdg_archive
if self.args.archive.startswith("/"):
self.archivpathbase = self.args.archive
self.xdgarchivpathbase = self.args.archive
else:
self.yesterpath = os.path.join(self.args.archive, 'workdir', self.yesterday.strftime("%Y-%m-%d"))
if not os.path.isdir(self.confdir):
os.mkdir(self.confdir)
with open(os.path.join(self.confdir, 'yesterpath'), 'w') as fh:
fh.write(self.yesterpath)
print("initial yesterpath is {}".format(self.yesterpath))
self.archivpathbase = os.path.join(os.environ['HOME'], self.args.archive)
self.xdgarchivpathbase = os.path.join('"$HOME', self.args.archive)
self._xdg()
self._give_me_today()
self.todaypath = self._checkdir(self._give_me_archivpath())
self.yesterdaypath = self._checkdir(self._give_me_archivpath(self._give_me_yesterday()))
print("initial yesterdaypath is {}".format(self.yesterdaypath))
print("initial todaypath is {}".format(self.todaypath))
# Helpers
# =======
def _full_path(self, partial):
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
if self.args.yearlydir:
path = os.path.join(os.environ['HOME'],
self.args.archive,"workdir",self.today.strftime("%Y"))
if self.args.monthlydir:
path = os.path.join(path, self.today.strftime("%m"))
def _xdg(self):
foundarchive=False
foundwork=False
xdguserdirs = os.environ['HOME']+'/.config/user-dirs.dirs'
try:
with fileinput.input(xdguserdirs, inplace=True) as fh:
for line in fh:
if line.startswith('XDG_ARCHIVE_DIR'):
print("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase, end='\n')
foundarchive=True
elif line.startswith('XDG_WORK_DIR'):
print("XDG_WORK_DIR=\"$HOME/" + self.args.mountpoint +'"', end='\n')
foundwork=True
else:
path = os.path.join(os.environ['HOME'], self.args.archive, "workdir")
print(line, end='')
except:
print("File not existing, create it: {}".format(xdguserdirs))
if not foundarchive:
with open(xdguserdirs, 'a') as fh:
fh.write("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase + '\n')
if not foundwork:
with open(xdguserdirs, 'a') as fh:
fh.write('XDG_WORK_DIR=\"$HOME/' + self.args.mountpoint + '"\n')
def _give_me_today(self):
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
return self.today
def _give_me_yesterday(self):
if os.path.exists(self.configlast):
with open(self.configlast, 'r') as fh:
self.yesterday = datetime.strptime(fh.readline().strip(), "%Y-%m-%d")
else:
self.yesterday = self.today
self._checkdir(self.confdir)
if not os.path.isdir(self.confdir):
os.mkdir(self.confdir)
with open(self.configlast, 'w') as fh:
fh.write(self.yesterday.date().strftime("%Y-%m-%d"))
return self.yesterday
def _give_me_archivpath(self, _date=None):
archivpathbase = self.archivpathbase
if _date == None:
_date = self.today
if self.args.yearlydir:
archivpathbase = os.path.join(archivpathbase, _date.strftime("%Y"))
if self.args.monthlydir:
archivpathbase = os.path.join(archivpathbase, _date.strftime("%m"))
if self.args.weeklydir:
archivpathbase = os.path.join(archivpathbase, int(_date.strftime("%W"))+1)
self.archivpath = os.path.join(archivpathbase, _date.strftime("%Y-%m-%d"))
return self.archivpath
def _checkdir(self, path):
if not os.path.exists(path) and not os.path.isdir(path):
try:
os.makedirs(path, exist_ok=True)
print("Created directory {}".format(path), flush=True)
except Exception as e:
print("[-] Error while check dir and zip files: ", e)
# if yesterdaypath != None and todaypath != None:
# print("yesterdaypath: {}, todaypath: {}".format(yesterdaypath, todaypath))
# if todaypath != yesterdaypath:
# _zipfiles(yesterdaypath)
# #Create .yesterdaypath in archive with yesterdaypath inside
# wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
# print("New yesterdaypath is {}".format(todaypath))
# with open(os.path.join(wdconfigdir, 'yesterdaypath'), 'w+') as f:
# f.write(todaypath)
return path
def _full_path(self, partial):
#self._give_me_today()
path = self._give_me_archivpath()
self._give_me_yesterday()
if partial.startswith("/"):
partial = partial[1:]
with open(os.path.join(self.confdir, 'yesterpath'), 'w') as fh:
fh.write(self.yesterpath)
path = os.path.join(check_dir(
os.path.join(path, self.today.strftime("%Y-%m-%d")),
self.yesterpath),
path = os.path.join(
self._checkdir( self._give_me_archivpath(self._give_me_today())),
partial
)
if self.today > self.yesterday:
if self.today.date() > self.yesterday.date():
self._cleanup_dirs()
print("zip yesterday: ", self.yesterdaypath)
self.yesterday = self.today
self.yesterdaypath = self.todaypath
with open(self.configlast, 'w') as fh:
fh.write(self.today.strftime("%Y-%m-%d"))
# print("yesterdaypath: {}, todaypath: {}".format(self.yesterdaypath, self.todaypath))
# print("yesterday: {}, today: {}".format(self.yesterday.date(), self.today.date()))
return path
def _cleanup_dirs(self):
print("Cleanup dir", self.yesterdaypath)
zip_fileext=".gz"
zip_compressionlevel=5
for root, dirs, files in os.walk(self.yesterdaypath, topdown=False):
for d in dirs:
print("cleanup",os.path.join(root, d))
if not _dir == self.todaypath and not os.listdir(os.path.join(root, d)):
print("Directory is empty -> remove it",
os.path.join(root, d))
os.remove(os.path.join(root, d))
for f in files:
print("compress", os.path.join(root, f))
if zip_fileext not in f:
try:
with open(os.path.join(root, f), 'rb') as f_in:
with gzip.open(
os.path.join(root, f+zip_fileext),
'wb',
compresslevel=zip_compressionlevel) as f_out:
shutil.copyfileobj(f_in, f_out)
except Exception as e:
print("Error during zipping file {}".format(os.path.join(root, f)), e)
traceback.print_exc()
else:
os.remove(os.path.join(root, f))
# Filesystem methods
# ==================
@ -191,77 +309,7 @@ class WorkdirFS(Operations):
def fsync(self, path, fdatasync, fh):
return self.flush(path, fh)
def cleanup_dirs(root):
today = datetime.now() - timedelta(hours=2)
today = today.strftime("%Y-%m-%d")
for root, dirs, files in os.walk(root, topdown=False):
for _dir in dirs:
print("cleanup",os.path.join(root, _dir))
if not _dir == today and not os.listdir(os.path.join(root, _dir)):
print("""Directory is empty -> remove it (not now implemented for
testpurpose)""",
os.path.join(root, _dir))
def check_dir(path, yesterpath=None):
if not os.path.exists(path) and not os.path.isdir(path):
#try:
os.makedirs(path, exist_ok=True)
print("Created directory {}".format(path), flush=True)
print("yesterpath: {} todaypath: {}".format(yesterpath, path))
if yesterpath != None:
if path != yesterpath:
_zipfiles(yesterpath)
#Create .yesterpath in archive with yesterpath inside
wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
#with open(os.path.join(wdconfigdir, 'yesterpath'), 'w+') as f:
# f.write(path)
f = open(os.path.join(wdconfigdir, 'yesterpath'), 'w+')
f.write(path)
f.flush()
f.close()
print("update yesterpath")
#except Exception as e:
# print("[-] Error while check dir and zip files: ", e)
return path
def _zipfiles(path):
print("Zip files in yesterdays archivdir {}".format(path))
zip_fileext=".gz"
#zip_compression=zipfile.ZIP_DEFLATED
zip_compressionlevel=5
files = []
# r=root, d=directories, f = files
for r, d, f in os.walk(path):
for file in f:
if zip_fileext not in file:
files.append(os.path.join(r, file))
for f in files:
print("file to zip: {} -> {}".format(os.path.basename(f), f+'.bzip'))
try:
with open(f, 'rb') as f_in:
with gzip.open(
f+zip_fileext,
'wb',
compresslevel=zip_compressionlevel) as f_out:
shutil.copyfileobj(f_in, f_out)
# with ZipFile(
# f+zip_fileext,
# 'w',
# allowZip64=True,
# compression=zip_compression,
# compresslevel=zip_compressionlevel
# ) as zf:
# zf.write(f, os.path.basename(f))
except Exception as e:
print("Error during zipping file {}".format(f), e)
else:
os.remove(f)
@ -269,43 +317,6 @@ def _zipfiles(path):
def main(args):
#FUSE(WorkdirFS(root), mountpoint, nothreads=True, foreground=True)
check_dir(os.path.join(os.environ['HOME'], args.mountpoint))
if args.archive[:0] == "/":
# Archive-Path is absolute path -> use it as is
check_dir(args.archive)
cleanup_dirs(args.archive)
xdg_entry = '"' + args.archive + '"'
else:
# Archive-path is relative, so use is as relative path from users
# homedir
check_dir(os.path.join(os.environ['HOME'], args.archive))
cleanup_dirs(os.path.join(os.environ['HOME'], args.archive))
xdg_entry = '"$HOME/' + args.archive + '"'
# first search if configuration exists for xdg-userdirs
# to use it with alias gowork and goarchive
foundarchive=False
foundwork=False
try:
with fileinput.input(os.environ['HOME']+'/.config/user-dirs.dirs',
inplace=True) as fh:
for line in fh:
if line.startswith('XDG_ARCHIVE_DIR'):
print("XDG_ARCHIVE_DIR=" + xdg_entry, end='\n')
foundarchive=True
elif line.startswith('XDG_WORK_DIR'):
print("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"', end='\n')
foundwork=True
else:
print(line, end='')
except:
print("File not existing, create it: {}".format(os.environ['HOME']+'/.config/user-dirs.dirs'))
if not foundarchive:
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
fh.write("XDG_ARCHIVE_DIR=" + xdg_entry + '\n')
if not foundwork:
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
fh.write('XDG_WORK_DIR=\"$HOME/' + args.mountpoint + '"\n')
# start FUSE filesystem
FUSE(WorkdirFS(args), os.path.join(os.environ['HOME'], args.mountpoint), nothreads=True, foreground=True)
@ -313,15 +324,16 @@ if __name__ == '__main__':
#main(sys.argv[2], sys.argv[1])
parser = argparse.ArgumentParser()
parser.add_argument("-a", "--archive",
default='archive', help="Path to archivedir-base")
default='archive/workdir', help="Path to archivedir-base")
parser.add_argument("-m", "--mountpoint",
default='Work', help='Path to Workdir')
parser.add_argument("-t", "--timeoffset", type=int, default=2, help="""If you're working
all day till 3 o'clock in the morning, set it to 4, so next day
archive-dir will be created 4 hours after midnight. You have 1h
tolerance, if you're working one day a little bit longer""")
parser.add_argument("-y", "--yearlydir", action="store_true")
parser.add_argument("-Y", "--yearlydir", action="store_true")
parser.add_argument("-M", "--monthlydir", action="store_true")
parser.add_argument("-W", "--weeklydir", action="store_true")
args = parser.parse_args()
print(args)
#root = os.environ['HOME']+'/archive'