cleanup code
This commit is contained in:
parent
73a6f034a5
commit
2f7098a866
1 changed files with 148 additions and 136 deletions
284
workdirfs.py
284
workdirfs.py
|
|
@ -6,7 +6,7 @@ import os
|
|||
import sys
|
||||
import errno
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, date
|
||||
import time
|
||||
|
||||
import fileinput
|
||||
|
|
@ -14,6 +14,8 @@ import fileinput
|
|||
import argparse
|
||||
import gzip
|
||||
import shutil
|
||||
import json
|
||||
import traceback
|
||||
|
||||
try:
|
||||
from fuse import FUSE, FuseOSError, Operations
|
||||
|
|
@ -28,51 +30,167 @@ except:
|
|||
class WorkdirFS(Operations):
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
|
||||
self.yesterday = datetime.now() - timedelta(hours=self.args.timeoffset)
|
||||
self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
||||
|
||||
if os.path.exists(os.path.join(self.confdir, 'yesterpath')):
|
||||
with open(os.path.join(self.confdir, 'yesterpath'), 'r') as fh:
|
||||
self.yesterpath = fh.readline().strip()
|
||||
# init configdir and configlast
|
||||
self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
||||
self.configlast = os.path.join(self.confdir, 'yesterday')
|
||||
|
||||
self._checkdir(self.confdir)
|
||||
|
||||
# init archivbase and xdg_archive
|
||||
if self.args.archive.startswith("/"):
|
||||
self.archivpathbase = self.args.archive
|
||||
self.xdgarchivpathbase = self.args.archive
|
||||
else:
|
||||
self.yesterpath = os.path.join(self.args.archive, 'workdir', self.yesterday.strftime("%Y-%m-%d"))
|
||||
if not os.path.isdir(self.confdir):
|
||||
os.mkdir(self.confdir)
|
||||
with open(os.path.join(self.confdir, 'yesterpath'), 'w') as fh:
|
||||
fh.write(self.yesterpath)
|
||||
print("initial yesterpath is {}".format(self.yesterpath))
|
||||
self.archivpathbase = os.path.join(os.environ['HOME'], self.args.archive)
|
||||
self.xdgarchivpathbase = os.path.join('"$HOME', self.args.archive)
|
||||
|
||||
self._xdg()
|
||||
self._give_me_today()
|
||||
self.todaypath = self._checkdir(self._give_me_archivpath())
|
||||
self.yesterdaypath = self._checkdir(self._give_me_archivpath(self._give_me_yesterday()))
|
||||
|
||||
print("initial yesterdaypath is {}".format(self.yesterdaypath))
|
||||
print("initial todaypath is {}".format(self.todaypath))
|
||||
|
||||
# Helpers
|
||||
# =======
|
||||
|
||||
def _full_path(self, partial):
|
||||
def _xdg(self):
|
||||
foundarchive=False
|
||||
foundwork=False
|
||||
xdguserdirs = os.environ['HOME']+'/.config/user-dirs.dirs'
|
||||
try:
|
||||
with fileinput.input(xdguserdirs, inplace=True) as fh:
|
||||
for line in fh:
|
||||
if line.startswith('XDG_ARCHIVE_DIR'):
|
||||
print("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase, end='\n')
|
||||
foundarchive=True
|
||||
elif line.startswith('XDG_WORK_DIR'):
|
||||
print("XDG_WORK_DIR=\"$HOME/" + self.args.mountpoint +'"', end='\n')
|
||||
foundwork=True
|
||||
else:
|
||||
print(line, end='')
|
||||
except:
|
||||
print("File not existing, create it: {}".format(xdguserdirs))
|
||||
|
||||
if not foundarchive:
|
||||
with open(xdguserdirs, 'a') as fh:
|
||||
fh.write("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase + '\n')
|
||||
if not foundwork:
|
||||
with open(xdguserdirs, 'a') as fh:
|
||||
fh.write('XDG_WORK_DIR=\"$HOME/' + self.args.mountpoint + '"\n')
|
||||
|
||||
def _give_me_today(self):
|
||||
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
|
||||
if self.args.yearlydir:
|
||||
path = os.path.join(os.environ['HOME'],
|
||||
self.args.archive,"workdir",self.today.strftime("%Y"))
|
||||
if self.args.monthlydir:
|
||||
path = os.path.join(path, self.today.strftime("%m"))
|
||||
return self.today
|
||||
|
||||
def _give_me_yesterday(self):
|
||||
if os.path.exists(self.configlast):
|
||||
with open(self.configlast, 'r') as fh:
|
||||
self.yesterday = datetime.strptime(fh.readline().strip(), "%Y-%m-%d")
|
||||
else:
|
||||
path = os.path.join(os.environ['HOME'], self.args.archive, "workdir")
|
||||
self.yesterday = self.today
|
||||
self._checkdir(self.confdir)
|
||||
if not os.path.isdir(self.confdir):
|
||||
os.mkdir(self.confdir)
|
||||
with open(self.configlast, 'w') as fh:
|
||||
fh.write(self.yesterday.date().strftime("%Y-%m-%d"))
|
||||
return self.yesterday
|
||||
|
||||
def _give_me_archivpath(self, _date=None):
|
||||
archivpathbase = self.archivpathbase
|
||||
if _date == None:
|
||||
_date = self.today
|
||||
if self.args.yearlydir:
|
||||
archivpathbase = os.path.join(archivpathbase, _date.strftime("%Y"))
|
||||
if self.args.monthlydir:
|
||||
archivpathbase = os.path.join(archivpathbase, _date.strftime("%m"))
|
||||
if self.args.weeklydir:
|
||||
archivpathbase = os.path.join(archivpathbase, int(_date.strftime("%W"))+1)
|
||||
self.archivpath = os.path.join(archivpathbase, _date.strftime("%Y-%m-%d"))
|
||||
return self.archivpath
|
||||
|
||||
def _checkdir(self, path):
|
||||
if not os.path.exists(path) and not os.path.isdir(path):
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
print("Created directory {}".format(path), flush=True)
|
||||
except Exception as e:
|
||||
print("[-] Error while check dir and zip files: ", e)
|
||||
|
||||
# if yesterdaypath != None and todaypath != None:
|
||||
# print("yesterdaypath: {}, todaypath: {}".format(yesterdaypath, todaypath))
|
||||
# if todaypath != yesterdaypath:
|
||||
# _zipfiles(yesterdaypath)
|
||||
# #Create .yesterdaypath in archive with yesterdaypath inside
|
||||
# wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
||||
# print("New yesterdaypath is {}".format(todaypath))
|
||||
# with open(os.path.join(wdconfigdir, 'yesterdaypath'), 'w+') as f:
|
||||
# f.write(todaypath)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def _full_path(self, partial):
|
||||
|
||||
#self._give_me_today()
|
||||
path = self._give_me_archivpath()
|
||||
self._give_me_yesterday()
|
||||
|
||||
if partial.startswith("/"):
|
||||
partial = partial[1:]
|
||||
|
||||
with open(os.path.join(self.confdir, 'yesterpath'), 'w') as fh:
|
||||
fh.write(self.yesterpath)
|
||||
|
||||
path = os.path.join(check_dir(
|
||||
os.path.join(path, self.today.strftime("%Y-%m-%d")),
|
||||
self.yesterpath),
|
||||
partial
|
||||
path = os.path.join(
|
||||
self._checkdir( self._give_me_archivpath(self._give_me_today())),
|
||||
partial
|
||||
)
|
||||
|
||||
if self.today > self.yesterday:
|
||||
if self.today.date() > self.yesterday.date():
|
||||
self._cleanup_dirs()
|
||||
print("zip yesterday: ", self.yesterdaypath)
|
||||
self.yesterday = self.today
|
||||
self.yesterdaypath = self.todaypath
|
||||
|
||||
with open(self.configlast, 'w') as fh:
|
||||
fh.write(self.today.strftime("%Y-%m-%d"))
|
||||
|
||||
# print("yesterdaypath: {}, todaypath: {}".format(self.yesterdaypath, self.todaypath))
|
||||
# print("yesterday: {}, today: {}".format(self.yesterday.date(), self.today.date()))
|
||||
return path
|
||||
|
||||
|
||||
def _cleanup_dirs(self):
|
||||
|
||||
print("Cleanup dir", self.yesterdaypath)
|
||||
zip_fileext=".gz"
|
||||
zip_compressionlevel=5
|
||||
for root, dirs, files in os.walk(self.yesterdaypath, topdown=False):
|
||||
for d in dirs:
|
||||
print("cleanup",os.path.join(root, d))
|
||||
if not _dir == self.todaypath and not os.listdir(os.path.join(root, d)):
|
||||
print("Directory is empty -> remove it",
|
||||
os.path.join(root, d))
|
||||
os.remove(os.path.join(root, d))
|
||||
for f in files:
|
||||
print("compress", os.path.join(root, f))
|
||||
if zip_fileext not in f:
|
||||
try:
|
||||
with open(os.path.join(root, f), 'rb') as f_in:
|
||||
with gzip.open(
|
||||
os.path.join(root, f+zip_fileext),
|
||||
'wb',
|
||||
compresslevel=zip_compressionlevel) as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
|
||||
except Exception as e:
|
||||
print("Error during zipping file {}".format(os.path.join(root, f)), e)
|
||||
traceback.print_exc()
|
||||
else:
|
||||
os.remove(os.path.join(root, f))
|
||||
|
||||
|
||||
# Filesystem methods
|
||||
# ==================
|
||||
|
||||
|
|
@ -191,77 +309,7 @@ class WorkdirFS(Operations):
|
|||
def fsync(self, path, fdatasync, fh):
|
||||
return self.flush(path, fh)
|
||||
|
||||
def cleanup_dirs(root):
|
||||
|
||||
today = datetime.now() - timedelta(hours=2)
|
||||
today = today.strftime("%Y-%m-%d")
|
||||
|
||||
for root, dirs, files in os.walk(root, topdown=False):
|
||||
for _dir in dirs:
|
||||
print("cleanup",os.path.join(root, _dir))
|
||||
if not _dir == today and not os.listdir(os.path.join(root, _dir)):
|
||||
print("""Directory is empty -> remove it (not now implemented for
|
||||
testpurpose)""",
|
||||
os.path.join(root, _dir))
|
||||
|
||||
|
||||
def check_dir(path, yesterpath=None):
|
||||
if not os.path.exists(path) and not os.path.isdir(path):
|
||||
#try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
print("Created directory {}".format(path), flush=True)
|
||||
print("yesterpath: {} todaypath: {}".format(yesterpath, path))
|
||||
if yesterpath != None:
|
||||
if path != yesterpath:
|
||||
_zipfiles(yesterpath)
|
||||
#Create .yesterpath in archive with yesterpath inside
|
||||
wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
||||
#with open(os.path.join(wdconfigdir, 'yesterpath'), 'w+') as f:
|
||||
# f.write(path)
|
||||
f = open(os.path.join(wdconfigdir, 'yesterpath'), 'w+')
|
||||
f.write(path)
|
||||
f.flush()
|
||||
f.close()
|
||||
print("update yesterpath")
|
||||
|
||||
#except Exception as e:
|
||||
# print("[-] Error while check dir and zip files: ", e)
|
||||
return path
|
||||
|
||||
def _zipfiles(path):
|
||||
print("Zip files in yesterdays archivdir {}".format(path))
|
||||
zip_fileext=".gz"
|
||||
#zip_compression=zipfile.ZIP_DEFLATED
|
||||
zip_compressionlevel=5
|
||||
files = []
|
||||
# r=root, d=directories, f = files
|
||||
for r, d, f in os.walk(path):
|
||||
for file in f:
|
||||
if zip_fileext not in file:
|
||||
files.append(os.path.join(r, file))
|
||||
|
||||
for f in files:
|
||||
print("file to zip: {} -> {}".format(os.path.basename(f), f+'.bzip'))
|
||||
try:
|
||||
with open(f, 'rb') as f_in:
|
||||
with gzip.open(
|
||||
f+zip_fileext,
|
||||
'wb',
|
||||
compresslevel=zip_compressionlevel) as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
|
||||
# with ZipFile(
|
||||
# f+zip_fileext,
|
||||
# 'w',
|
||||
# allowZip64=True,
|
||||
# compression=zip_compression,
|
||||
# compresslevel=zip_compressionlevel
|
||||
# ) as zf:
|
||||
# zf.write(f, os.path.basename(f))
|
||||
except Exception as e:
|
||||
print("Error during zipping file {}".format(f), e)
|
||||
else:
|
||||
os.remove(f)
|
||||
|
||||
|
||||
|
||||
|
|
@ -269,43 +317,6 @@ def _zipfiles(path):
|
|||
|
||||
def main(args):
|
||||
#FUSE(WorkdirFS(root), mountpoint, nothreads=True, foreground=True)
|
||||
check_dir(os.path.join(os.environ['HOME'], args.mountpoint))
|
||||
if args.archive[:0] == "/":
|
||||
# Archive-Path is absolute path -> use it as is
|
||||
check_dir(args.archive)
|
||||
cleanup_dirs(args.archive)
|
||||
xdg_entry = '"' + args.archive + '"'
|
||||
else:
|
||||
# Archive-path is relative, so use is as relative path from users
|
||||
# homedir
|
||||
check_dir(os.path.join(os.environ['HOME'], args.archive))
|
||||
cleanup_dirs(os.path.join(os.environ['HOME'], args.archive))
|
||||
xdg_entry = '"$HOME/' + args.archive + '"'
|
||||
# first search if configuration exists for xdg-userdirs
|
||||
# to use it with alias gowork and goarchive
|
||||
foundarchive=False
|
||||
foundwork=False
|
||||
try:
|
||||
with fileinput.input(os.environ['HOME']+'/.config/user-dirs.dirs',
|
||||
inplace=True) as fh:
|
||||
for line in fh:
|
||||
if line.startswith('XDG_ARCHIVE_DIR'):
|
||||
print("XDG_ARCHIVE_DIR=" + xdg_entry, end='\n')
|
||||
foundarchive=True
|
||||
elif line.startswith('XDG_WORK_DIR'):
|
||||
print("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"', end='\n')
|
||||
foundwork=True
|
||||
else:
|
||||
print(line, end='')
|
||||
except:
|
||||
print("File not existing, create it: {}".format(os.environ['HOME']+'/.config/user-dirs.dirs'))
|
||||
if not foundarchive:
|
||||
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
|
||||
fh.write("XDG_ARCHIVE_DIR=" + xdg_entry + '\n')
|
||||
if not foundwork:
|
||||
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
|
||||
fh.write('XDG_WORK_DIR=\"$HOME/' + args.mountpoint + '"\n')
|
||||
|
||||
# start FUSE filesystem
|
||||
FUSE(WorkdirFS(args), os.path.join(os.environ['HOME'], args.mountpoint), nothreads=True, foreground=True)
|
||||
|
||||
|
|
@ -313,15 +324,16 @@ if __name__ == '__main__':
|
|||
#main(sys.argv[2], sys.argv[1])
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-a", "--archive",
|
||||
default='archive', help="Path to archivedir-base")
|
||||
default='archive/workdir', help="Path to archivedir-base")
|
||||
parser.add_argument("-m", "--mountpoint",
|
||||
default='Work', help='Path to Workdir')
|
||||
parser.add_argument("-t", "--timeoffset", type=int, default=2, help="""If you're working
|
||||
all day till 3 o'clock in the morning, set it to 4, so next day
|
||||
archive-dir will be created 4 hours after midnight. You have 1h
|
||||
tolerance, if you're working one day a little bit longer""")
|
||||
parser.add_argument("-y", "--yearlydir", action="store_true")
|
||||
parser.add_argument("-Y", "--yearlydir", action="store_true")
|
||||
parser.add_argument("-M", "--monthlydir", action="store_true")
|
||||
parser.add_argument("-W", "--weeklydir", action="store_true")
|
||||
args = parser.parse_args()
|
||||
print(args)
|
||||
#root = os.environ['HOME']+'/archive'
|
||||
|
|
|
|||
Loading…
Reference in a new issue