cleanup code
This commit is contained in:
parent
73a6f034a5
commit
2f7098a866
1 changed files with 148 additions and 136 deletions
284
workdirfs.py
284
workdirfs.py
|
|
@ -6,7 +6,7 @@ import os
|
||||||
import sys
|
import sys
|
||||||
import errno
|
import errno
|
||||||
|
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta, date
|
||||||
import time
|
import time
|
||||||
|
|
||||||
import fileinput
|
import fileinput
|
||||||
|
|
@ -14,6 +14,8 @@ import fileinput
|
||||||
import argparse
|
import argparse
|
||||||
import gzip
|
import gzip
|
||||||
import shutil
|
import shutil
|
||||||
|
import json
|
||||||
|
import traceback
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from fuse import FUSE, FuseOSError, Operations
|
from fuse import FUSE, FuseOSError, Operations
|
||||||
|
|
@ -28,51 +30,167 @@ except:
|
||||||
class WorkdirFS(Operations):
|
class WorkdirFS(Operations):
|
||||||
def __init__(self, args):
|
def __init__(self, args):
|
||||||
self.args = args
|
self.args = args
|
||||||
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
|
|
||||||
self.yesterday = datetime.now() - timedelta(hours=self.args.timeoffset)
|
|
||||||
self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
|
||||||
|
|
||||||
if os.path.exists(os.path.join(self.confdir, 'yesterpath')):
|
# init configdir and configlast
|
||||||
with open(os.path.join(self.confdir, 'yesterpath'), 'r') as fh:
|
self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
||||||
self.yesterpath = fh.readline().strip()
|
self.configlast = os.path.join(self.confdir, 'yesterday')
|
||||||
|
|
||||||
|
self._checkdir(self.confdir)
|
||||||
|
|
||||||
|
# init archivbase and xdg_archive
|
||||||
|
if self.args.archive.startswith("/"):
|
||||||
|
self.archivpathbase = self.args.archive
|
||||||
|
self.xdgarchivpathbase = self.args.archive
|
||||||
else:
|
else:
|
||||||
self.yesterpath = os.path.join(self.args.archive, 'workdir', self.yesterday.strftime("%Y-%m-%d"))
|
self.archivpathbase = os.path.join(os.environ['HOME'], self.args.archive)
|
||||||
if not os.path.isdir(self.confdir):
|
self.xdgarchivpathbase = os.path.join('"$HOME', self.args.archive)
|
||||||
os.mkdir(self.confdir)
|
|
||||||
with open(os.path.join(self.confdir, 'yesterpath'), 'w') as fh:
|
self._xdg()
|
||||||
fh.write(self.yesterpath)
|
self._give_me_today()
|
||||||
print("initial yesterpath is {}".format(self.yesterpath))
|
self.todaypath = self._checkdir(self._give_me_archivpath())
|
||||||
|
self.yesterdaypath = self._checkdir(self._give_me_archivpath(self._give_me_yesterday()))
|
||||||
|
|
||||||
|
print("initial yesterdaypath is {}".format(self.yesterdaypath))
|
||||||
|
print("initial todaypath is {}".format(self.todaypath))
|
||||||
|
|
||||||
# Helpers
|
# Helpers
|
||||||
# =======
|
# =======
|
||||||
|
|
||||||
def _full_path(self, partial):
|
def _xdg(self):
|
||||||
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
|
foundarchive=False
|
||||||
if self.args.yearlydir:
|
foundwork=False
|
||||||
path = os.path.join(os.environ['HOME'],
|
xdguserdirs = os.environ['HOME']+'/.config/user-dirs.dirs'
|
||||||
self.args.archive,"workdir",self.today.strftime("%Y"))
|
try:
|
||||||
if self.args.monthlydir:
|
with fileinput.input(xdguserdirs, inplace=True) as fh:
|
||||||
path = os.path.join(path, self.today.strftime("%m"))
|
for line in fh:
|
||||||
|
if line.startswith('XDG_ARCHIVE_DIR'):
|
||||||
|
print("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase, end='\n')
|
||||||
|
foundarchive=True
|
||||||
|
elif line.startswith('XDG_WORK_DIR'):
|
||||||
|
print("XDG_WORK_DIR=\"$HOME/" + self.args.mountpoint +'"', end='\n')
|
||||||
|
foundwork=True
|
||||||
else:
|
else:
|
||||||
path = os.path.join(os.environ['HOME'], self.args.archive, "workdir")
|
print(line, end='')
|
||||||
|
except:
|
||||||
|
print("File not existing, create it: {}".format(xdguserdirs))
|
||||||
|
|
||||||
|
if not foundarchive:
|
||||||
|
with open(xdguserdirs, 'a') as fh:
|
||||||
|
fh.write("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase + '\n')
|
||||||
|
if not foundwork:
|
||||||
|
with open(xdguserdirs, 'a') as fh:
|
||||||
|
fh.write('XDG_WORK_DIR=\"$HOME/' + self.args.mountpoint + '"\n')
|
||||||
|
|
||||||
|
def _give_me_today(self):
|
||||||
|
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
|
||||||
|
return self.today
|
||||||
|
|
||||||
|
def _give_me_yesterday(self):
|
||||||
|
if os.path.exists(self.configlast):
|
||||||
|
with open(self.configlast, 'r') as fh:
|
||||||
|
self.yesterday = datetime.strptime(fh.readline().strip(), "%Y-%m-%d")
|
||||||
|
else:
|
||||||
|
self.yesterday = self.today
|
||||||
|
self._checkdir(self.confdir)
|
||||||
|
if not os.path.isdir(self.confdir):
|
||||||
|
os.mkdir(self.confdir)
|
||||||
|
with open(self.configlast, 'w') as fh:
|
||||||
|
fh.write(self.yesterday.date().strftime("%Y-%m-%d"))
|
||||||
|
return self.yesterday
|
||||||
|
|
||||||
|
def _give_me_archivpath(self, _date=None):
|
||||||
|
archivpathbase = self.archivpathbase
|
||||||
|
if _date == None:
|
||||||
|
_date = self.today
|
||||||
|
if self.args.yearlydir:
|
||||||
|
archivpathbase = os.path.join(archivpathbase, _date.strftime("%Y"))
|
||||||
|
if self.args.monthlydir:
|
||||||
|
archivpathbase = os.path.join(archivpathbase, _date.strftime("%m"))
|
||||||
|
if self.args.weeklydir:
|
||||||
|
archivpathbase = os.path.join(archivpathbase, int(_date.strftime("%W"))+1)
|
||||||
|
self.archivpath = os.path.join(archivpathbase, _date.strftime("%Y-%m-%d"))
|
||||||
|
return self.archivpath
|
||||||
|
|
||||||
|
def _checkdir(self, path):
|
||||||
|
if not os.path.exists(path) and not os.path.isdir(path):
|
||||||
|
try:
|
||||||
|
os.makedirs(path, exist_ok=True)
|
||||||
|
print("Created directory {}".format(path), flush=True)
|
||||||
|
except Exception as e:
|
||||||
|
print("[-] Error while check dir and zip files: ", e)
|
||||||
|
|
||||||
|
# if yesterdaypath != None and todaypath != None:
|
||||||
|
# print("yesterdaypath: {}, todaypath: {}".format(yesterdaypath, todaypath))
|
||||||
|
# if todaypath != yesterdaypath:
|
||||||
|
# _zipfiles(yesterdaypath)
|
||||||
|
# #Create .yesterdaypath in archive with yesterdaypath inside
|
||||||
|
# wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
||||||
|
# print("New yesterdaypath is {}".format(todaypath))
|
||||||
|
# with open(os.path.join(wdconfigdir, 'yesterdaypath'), 'w+') as f:
|
||||||
|
# f.write(todaypath)
|
||||||
|
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _full_path(self, partial):
|
||||||
|
|
||||||
|
#self._give_me_today()
|
||||||
|
path = self._give_me_archivpath()
|
||||||
|
self._give_me_yesterday()
|
||||||
|
|
||||||
if partial.startswith("/"):
|
if partial.startswith("/"):
|
||||||
partial = partial[1:]
|
partial = partial[1:]
|
||||||
|
|
||||||
with open(os.path.join(self.confdir, 'yesterpath'), 'w') as fh:
|
|
||||||
fh.write(self.yesterpath)
|
|
||||||
|
|
||||||
path = os.path.join(check_dir(
|
path = os.path.join(
|
||||||
os.path.join(path, self.today.strftime("%Y-%m-%d")),
|
self._checkdir( self._give_me_archivpath(self._give_me_today())),
|
||||||
self.yesterpath),
|
|
||||||
partial
|
partial
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.today > self.yesterday:
|
if self.today.date() > self.yesterday.date():
|
||||||
|
self._cleanup_dirs()
|
||||||
|
print("zip yesterday: ", self.yesterdaypath)
|
||||||
self.yesterday = self.today
|
self.yesterday = self.today
|
||||||
|
self.yesterdaypath = self.todaypath
|
||||||
|
|
||||||
|
with open(self.configlast, 'w') as fh:
|
||||||
|
fh.write(self.today.strftime("%Y-%m-%d"))
|
||||||
|
|
||||||
|
# print("yesterdaypath: {}, todaypath: {}".format(self.yesterdaypath, self.todaypath))
|
||||||
|
# print("yesterday: {}, today: {}".format(self.yesterday.date(), self.today.date()))
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _cleanup_dirs(self):
|
||||||
|
|
||||||
|
print("Cleanup dir", self.yesterdaypath)
|
||||||
|
zip_fileext=".gz"
|
||||||
|
zip_compressionlevel=5
|
||||||
|
for root, dirs, files in os.walk(self.yesterdaypath, topdown=False):
|
||||||
|
for d in dirs:
|
||||||
|
print("cleanup",os.path.join(root, d))
|
||||||
|
if not _dir == self.todaypath and not os.listdir(os.path.join(root, d)):
|
||||||
|
print("Directory is empty -> remove it",
|
||||||
|
os.path.join(root, d))
|
||||||
|
os.remove(os.path.join(root, d))
|
||||||
|
for f in files:
|
||||||
|
print("compress", os.path.join(root, f))
|
||||||
|
if zip_fileext not in f:
|
||||||
|
try:
|
||||||
|
with open(os.path.join(root, f), 'rb') as f_in:
|
||||||
|
with gzip.open(
|
||||||
|
os.path.join(root, f+zip_fileext),
|
||||||
|
'wb',
|
||||||
|
compresslevel=zip_compressionlevel) as f_out:
|
||||||
|
shutil.copyfileobj(f_in, f_out)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print("Error during zipping file {}".format(os.path.join(root, f)), e)
|
||||||
|
traceback.print_exc()
|
||||||
|
else:
|
||||||
|
os.remove(os.path.join(root, f))
|
||||||
|
|
||||||
|
|
||||||
# Filesystem methods
|
# Filesystem methods
|
||||||
# ==================
|
# ==================
|
||||||
|
|
||||||
|
|
@ -191,77 +309,7 @@ class WorkdirFS(Operations):
|
||||||
def fsync(self, path, fdatasync, fh):
|
def fsync(self, path, fdatasync, fh):
|
||||||
return self.flush(path, fh)
|
return self.flush(path, fh)
|
||||||
|
|
||||||
def cleanup_dirs(root):
|
|
||||||
|
|
||||||
today = datetime.now() - timedelta(hours=2)
|
|
||||||
today = today.strftime("%Y-%m-%d")
|
|
||||||
|
|
||||||
for root, dirs, files in os.walk(root, topdown=False):
|
|
||||||
for _dir in dirs:
|
|
||||||
print("cleanup",os.path.join(root, _dir))
|
|
||||||
if not _dir == today and not os.listdir(os.path.join(root, _dir)):
|
|
||||||
print("""Directory is empty -> remove it (not now implemented for
|
|
||||||
testpurpose)""",
|
|
||||||
os.path.join(root, _dir))
|
|
||||||
|
|
||||||
|
|
||||||
def check_dir(path, yesterpath=None):
|
|
||||||
if not os.path.exists(path) and not os.path.isdir(path):
|
|
||||||
#try:
|
|
||||||
os.makedirs(path, exist_ok=True)
|
|
||||||
print("Created directory {}".format(path), flush=True)
|
|
||||||
print("yesterpath: {} todaypath: {}".format(yesterpath, path))
|
|
||||||
if yesterpath != None:
|
|
||||||
if path != yesterpath:
|
|
||||||
_zipfiles(yesterpath)
|
|
||||||
#Create .yesterpath in archive with yesterpath inside
|
|
||||||
wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
|
||||||
#with open(os.path.join(wdconfigdir, 'yesterpath'), 'w+') as f:
|
|
||||||
# f.write(path)
|
|
||||||
f = open(os.path.join(wdconfigdir, 'yesterpath'), 'w+')
|
|
||||||
f.write(path)
|
|
||||||
f.flush()
|
|
||||||
f.close()
|
|
||||||
print("update yesterpath")
|
|
||||||
|
|
||||||
#except Exception as e:
|
|
||||||
# print("[-] Error while check dir and zip files: ", e)
|
|
||||||
return path
|
|
||||||
|
|
||||||
def _zipfiles(path):
|
|
||||||
print("Zip files in yesterdays archivdir {}".format(path))
|
|
||||||
zip_fileext=".gz"
|
|
||||||
#zip_compression=zipfile.ZIP_DEFLATED
|
|
||||||
zip_compressionlevel=5
|
|
||||||
files = []
|
|
||||||
# r=root, d=directories, f = files
|
|
||||||
for r, d, f in os.walk(path):
|
|
||||||
for file in f:
|
|
||||||
if zip_fileext not in file:
|
|
||||||
files.append(os.path.join(r, file))
|
|
||||||
|
|
||||||
for f in files:
|
|
||||||
print("file to zip: {} -> {}".format(os.path.basename(f), f+'.bzip'))
|
|
||||||
try:
|
|
||||||
with open(f, 'rb') as f_in:
|
|
||||||
with gzip.open(
|
|
||||||
f+zip_fileext,
|
|
||||||
'wb',
|
|
||||||
compresslevel=zip_compressionlevel) as f_out:
|
|
||||||
shutil.copyfileobj(f_in, f_out)
|
|
||||||
|
|
||||||
# with ZipFile(
|
|
||||||
# f+zip_fileext,
|
|
||||||
# 'w',
|
|
||||||
# allowZip64=True,
|
|
||||||
# compression=zip_compression,
|
|
||||||
# compresslevel=zip_compressionlevel
|
|
||||||
# ) as zf:
|
|
||||||
# zf.write(f, os.path.basename(f))
|
|
||||||
except Exception as e:
|
|
||||||
print("Error during zipping file {}".format(f), e)
|
|
||||||
else:
|
|
||||||
os.remove(f)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -269,43 +317,6 @@ def _zipfiles(path):
|
||||||
|
|
||||||
def main(args):
|
def main(args):
|
||||||
#FUSE(WorkdirFS(root), mountpoint, nothreads=True, foreground=True)
|
#FUSE(WorkdirFS(root), mountpoint, nothreads=True, foreground=True)
|
||||||
check_dir(os.path.join(os.environ['HOME'], args.mountpoint))
|
|
||||||
if args.archive[:0] == "/":
|
|
||||||
# Archive-Path is absolute path -> use it as is
|
|
||||||
check_dir(args.archive)
|
|
||||||
cleanup_dirs(args.archive)
|
|
||||||
xdg_entry = '"' + args.archive + '"'
|
|
||||||
else:
|
|
||||||
# Archive-path is relative, so use is as relative path from users
|
|
||||||
# homedir
|
|
||||||
check_dir(os.path.join(os.environ['HOME'], args.archive))
|
|
||||||
cleanup_dirs(os.path.join(os.environ['HOME'], args.archive))
|
|
||||||
xdg_entry = '"$HOME/' + args.archive + '"'
|
|
||||||
# first search if configuration exists for xdg-userdirs
|
|
||||||
# to use it with alias gowork and goarchive
|
|
||||||
foundarchive=False
|
|
||||||
foundwork=False
|
|
||||||
try:
|
|
||||||
with fileinput.input(os.environ['HOME']+'/.config/user-dirs.dirs',
|
|
||||||
inplace=True) as fh:
|
|
||||||
for line in fh:
|
|
||||||
if line.startswith('XDG_ARCHIVE_DIR'):
|
|
||||||
print("XDG_ARCHIVE_DIR=" + xdg_entry, end='\n')
|
|
||||||
foundarchive=True
|
|
||||||
elif line.startswith('XDG_WORK_DIR'):
|
|
||||||
print("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"', end='\n')
|
|
||||||
foundwork=True
|
|
||||||
else:
|
|
||||||
print(line, end='')
|
|
||||||
except:
|
|
||||||
print("File not existing, create it: {}".format(os.environ['HOME']+'/.config/user-dirs.dirs'))
|
|
||||||
if not foundarchive:
|
|
||||||
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
|
|
||||||
fh.write("XDG_ARCHIVE_DIR=" + xdg_entry + '\n')
|
|
||||||
if not foundwork:
|
|
||||||
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
|
|
||||||
fh.write('XDG_WORK_DIR=\"$HOME/' + args.mountpoint + '"\n')
|
|
||||||
|
|
||||||
# start FUSE filesystem
|
# start FUSE filesystem
|
||||||
FUSE(WorkdirFS(args), os.path.join(os.environ['HOME'], args.mountpoint), nothreads=True, foreground=True)
|
FUSE(WorkdirFS(args), os.path.join(os.environ['HOME'], args.mountpoint), nothreads=True, foreground=True)
|
||||||
|
|
||||||
|
|
@ -313,15 +324,16 @@ if __name__ == '__main__':
|
||||||
#main(sys.argv[2], sys.argv[1])
|
#main(sys.argv[2], sys.argv[1])
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("-a", "--archive",
|
parser.add_argument("-a", "--archive",
|
||||||
default='archive', help="Path to archivedir-base")
|
default='archive/workdir', help="Path to archivedir-base")
|
||||||
parser.add_argument("-m", "--mountpoint",
|
parser.add_argument("-m", "--mountpoint",
|
||||||
default='Work', help='Path to Workdir')
|
default='Work', help='Path to Workdir')
|
||||||
parser.add_argument("-t", "--timeoffset", type=int, default=2, help="""If you're working
|
parser.add_argument("-t", "--timeoffset", type=int, default=2, help="""If you're working
|
||||||
all day till 3 o'clock in the morning, set it to 4, so next day
|
all day till 3 o'clock in the morning, set it to 4, so next day
|
||||||
archive-dir will be created 4 hours after midnight. You have 1h
|
archive-dir will be created 4 hours after midnight. You have 1h
|
||||||
tolerance, if you're working one day a little bit longer""")
|
tolerance, if you're working one day a little bit longer""")
|
||||||
parser.add_argument("-y", "--yearlydir", action="store_true")
|
parser.add_argument("-Y", "--yearlydir", action="store_true")
|
||||||
parser.add_argument("-M", "--monthlydir", action="store_true")
|
parser.add_argument("-M", "--monthlydir", action="store_true")
|
||||||
|
parser.add_argument("-W", "--weeklydir", action="store_true")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
print(args)
|
print(args)
|
||||||
#root = os.environ['HOME']+'/archive'
|
#root = os.environ['HOME']+'/archive'
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue