Merge branch 'gzip-postcompress'
This commit is contained in:
commit
875343e0d7
1 changed files with 179 additions and 65 deletions
244
workdirfs.py
244
workdirfs.py
|
|
@ -6,11 +6,15 @@ import os
|
|||
import sys
|
||||
import errno
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, date
|
||||
import time
|
||||
|
||||
import fileinput
|
||||
import argparse
|
||||
import gzip
|
||||
import shutil
|
||||
import json
|
||||
import traceback
|
||||
|
||||
try:
|
||||
from fuse import FUSE, FuseOSError, Operations
|
||||
|
|
@ -25,28 +29,168 @@ except:
|
|||
class WorkdirFS(Operations):
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
|
||||
|
||||
# init configdir and configlast
|
||||
self.confdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
||||
self.configlast = os.path.join(self.confdir, 'yesterday')
|
||||
|
||||
self._checkdir(self.confdir)
|
||||
|
||||
# init archivbase and xdg_archive
|
||||
if self.args.archive.startswith("/"):
|
||||
self.archivpathbase = self.args.archive
|
||||
self.xdgarchivpathbase = self.args.archive
|
||||
else:
|
||||
self.archivpathbase = os.path.join(os.environ['HOME'], self.args.archive)
|
||||
self.xdgarchivpathbase = os.path.join('"$HOME', self.args.archive)
|
||||
|
||||
self._xdg()
|
||||
self._give_me_today()
|
||||
self.todaypath = self._checkdir(self._give_me_archivpath())
|
||||
self.yesterdaypath = self._checkdir(self._give_me_archivpath(self._give_me_yesterday()))
|
||||
|
||||
print("initial yesterdaypath is {}".format(self.yesterdaypath))
|
||||
print("initial todaypath is {}".format(self.todaypath))
|
||||
|
||||
# Helpers
|
||||
# =======
|
||||
|
||||
def _full_path(self, partial):
|
||||
def _xdg(self):
|
||||
foundarchive=False
|
||||
foundwork=False
|
||||
xdguserdirs = os.environ['HOME']+'/.config/user-dirs.dirs'
|
||||
try:
|
||||
with fileinput.input(xdguserdirs, inplace=True) as fh:
|
||||
for line in fh:
|
||||
if line.startswith('XDG_ARCHIVE_DIR'):
|
||||
print("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase, end='\n')
|
||||
foundarchive=True
|
||||
elif line.startswith('XDG_WORK_DIR'):
|
||||
print("XDG_WORK_DIR=\"$HOME/" + self.args.mountpoint +'"', end='\n')
|
||||
foundwork=True
|
||||
else:
|
||||
print(line, end='')
|
||||
except:
|
||||
print("File not existing, create it: {}".format(xdguserdirs))
|
||||
|
||||
if not foundarchive:
|
||||
with open(xdguserdirs, 'a') as fh:
|
||||
fh.write("XDG_ARCHIVE_DIR=" + self.xdgarchivpathbase + '\n')
|
||||
if not foundwork:
|
||||
with open(xdguserdirs, 'a') as fh:
|
||||
fh.write('XDG_WORK_DIR=\"$HOME/' + self.args.mountpoint + '"\n')
|
||||
|
||||
def _give_me_today(self):
|
||||
self.today = datetime.now() - timedelta(hours=self.args.timeoffset)
|
||||
if self.args.yearlydir:
|
||||
path = os.path.join(os.environ['HOME'],
|
||||
self.args.archive,"workdir",self.today.strftime("%Y"))
|
||||
if self.args.monthlydir:
|
||||
path = os.path.join(path, self.today.strftime("%m"))
|
||||
return self.today
|
||||
|
||||
def _give_me_yesterday(self):
|
||||
if os.path.exists(self.configlast):
|
||||
with open(self.configlast, 'r') as fh:
|
||||
self.yesterday = datetime.strptime(fh.readline().strip(), "%Y-%m-%d")
|
||||
else:
|
||||
path = os.path.join(os.environ['HOME'], self.args.archive, "workdir")
|
||||
self.yesterday = self.today
|
||||
self._checkdir(self.confdir)
|
||||
if not os.path.isdir(self.confdir):
|
||||
os.mkdir(self.confdir)
|
||||
with open(self.configlast, 'w') as fh:
|
||||
fh.write(self.yesterday.date().strftime("%Y-%m-%d"))
|
||||
return self.yesterday
|
||||
|
||||
def _give_me_archivpath(self, _date=None):
|
||||
archivpathbase = self.archivpathbase
|
||||
if _date == None:
|
||||
_date = self.today
|
||||
if self.args.yearlydir:
|
||||
archivpathbase = os.path.join(archivpathbase, _date.strftime("%Y"))
|
||||
if self.args.monthlydir:
|
||||
archivpathbase = os.path.join(archivpathbase, _date.strftime("%m"))
|
||||
if self.args.weeklydir:
|
||||
archivpathbase = os.path.join(archivpathbase, int(_date.strftime("%W"))+1)
|
||||
self.archivpath = os.path.join(archivpathbase, _date.strftime("%Y-%m-%d"))
|
||||
return self.archivpath
|
||||
|
||||
def _checkdir(self, path):
|
||||
if not os.path.exists(path) and not os.path.isdir(path):
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
print("Created directory {}".format(path), flush=True)
|
||||
except Exception as e:
|
||||
print("[-] Error while check dir and zip files: ", e)
|
||||
|
||||
# if yesterdaypath != None and todaypath != None:
|
||||
# print("yesterdaypath: {}, todaypath: {}".format(yesterdaypath, todaypath))
|
||||
# if todaypath != yesterdaypath:
|
||||
# _zipfiles(yesterdaypath)
|
||||
# #Create .yesterdaypath in archive with yesterdaypath inside
|
||||
# wdconfigdir = os.path.join(os.environ['HOME'], '.local', 'workdirfs')
|
||||
# print("New yesterdaypath is {}".format(todaypath))
|
||||
# with open(os.path.join(wdconfigdir, 'yesterdaypath'), 'w+') as f:
|
||||
# f.write(todaypath)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def _full_path(self, partial):
|
||||
|
||||
#self._give_me_today()
|
||||
path = self._give_me_archivpath()
|
||||
self._give_me_yesterday()
|
||||
|
||||
if partial.startswith("/"):
|
||||
partial = partial[1:]
|
||||
|
||||
path = os.path.join(check_dir(os.path.join(path, self.today.strftime("%Y-%m-%d"))), partial)
|
||||
|
||||
path = os.path.join(
|
||||
self._checkdir( self._give_me_archivpath(self._give_me_today())),
|
||||
partial
|
||||
)
|
||||
|
||||
if self.today.date() > self.yesterday.date():
|
||||
self._cleanup_dirs()
|
||||
print("zip yesterday: ", self.yesterdaypath)
|
||||
self.yesterday = self.today
|
||||
self.yesterdaypath = self.todaypath
|
||||
|
||||
with open(self.configlast, 'w') as fh:
|
||||
fh.write(self.today.strftime("%Y-%m-%d"))
|
||||
|
||||
# print("yesterdaypath: {}, todaypath: {}".format(self.yesterdaypath, self.todaypath))
|
||||
# print("yesterday: {}, today: {}".format(self.yesterday.date(), self.today.date()))
|
||||
return path
|
||||
|
||||
|
||||
def _cleanup_dirs(self):
|
||||
|
||||
print("Cleanup dir", self.yesterdaypath)
|
||||
zip_fileext=".gz"
|
||||
zip_compressionlevel=5
|
||||
for root, dirs, files in os.walk(self.yesterdaypath, topdown=False):
|
||||
for d in dirs:
|
||||
print("cleanup",os.path.join(root, d))
|
||||
if not _dir == self.todaypath and not os.listdir(os.path.join(root, d)):
|
||||
print("Directory is empty -> remove it",
|
||||
os.path.join(root, d))
|
||||
os.remove(os.path.join(root, d))
|
||||
if self.args.compress:
|
||||
for f in files:
|
||||
print("compress", os.path.join(root, f))
|
||||
if zip_fileext not in f:
|
||||
try:
|
||||
with open(os.path.join(root, f), 'rb') as f_in:
|
||||
with gzip.open(
|
||||
os.path.join(root, f+zip_fileext),
|
||||
'wb',
|
||||
compresslevel=zip_compressionlevel) as f_out:
|
||||
shutil.copyfileobj(f_in, f_out)
|
||||
|
||||
except Exception as e:
|
||||
print("Error during zipping file {}".format(os.path.join(root, f)), e)
|
||||
traceback.print_exc()
|
||||
else:
|
||||
os.remove(os.path.join(root, f))
|
||||
|
||||
|
||||
# Filesystem methods
|
||||
# ==================
|
||||
|
||||
|
|
@ -165,60 +309,14 @@ class WorkdirFS(Operations):
|
|||
def fsync(self, path, fdatasync, fh):
|
||||
return self.flush(path, fh)
|
||||
|
||||
def cleanup_dirs(root):
|
||||
|
||||
today = datetime.now() - timedelta(hours=2)
|
||||
today = today.strftime("%Y-%m-%d")
|
||||
|
||||
for root, dirs, files in os.walk(root, topdown=False):
|
||||
for _dir in dirs:
|
||||
print("cleanup",os.path.join(root, _dir))
|
||||
if not _dir == today and not os.listdir(os.path.join(root, _dir)):
|
||||
print("""Directory is empty -> remove it (not now implemented for
|
||||
testpurpose)""",
|
||||
os.path.join(root, _dir))
|
||||
|
||||
|
||||
def check_dir(path):
|
||||
checkdir = os.path.isdir(path)
|
||||
if not checkdir:
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
print("Created directory {}".format(path), flush=True)
|
||||
except:
|
||||
print("[-] Makedir error")
|
||||
return path
|
||||
|
||||
|
||||
|
||||
|
||||
def main(args):
|
||||
#FUSE(WorkdirFS(root), mountpoint, nothreads=True, foreground=True)
|
||||
check_dir(os.path.join(os.environ['HOME'], args.archive))
|
||||
check_dir(os.path.join(os.environ['HOME'], args.mountpoint))
|
||||
cleanup_dirs(os.path.join(os.environ['HOME'], args.archive))
|
||||
# first search if configuration exists for xdg-userdirs
|
||||
# to use it with alias gowork and goarchive
|
||||
foundarchive=False
|
||||
foundwork=False
|
||||
try:
|
||||
with fileinput.input(os.environ['HOME']+'/.config/user-dirs.dirs',
|
||||
inplace=True) as fh:
|
||||
for line in fh:
|
||||
if line.startswith('XDG_ARCHIVE_DIR'):
|
||||
print("XDG_ARCHIVE_DIR=\"$HOME/"+args.archive+'"', end='\n')
|
||||
foundarchive=True
|
||||
elif line.startswith('XDG_WORK_DIR'):
|
||||
print("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"', end='\n')
|
||||
foundwork=True
|
||||
else:
|
||||
print(line, end='')
|
||||
except:
|
||||
print("File not existing, create it: {}".format(os.environ['HOME']+'/.config/user-dirs.dirs'))
|
||||
if not foundarchive:
|
||||
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
|
||||
fh.write("XDG_ARCHIVE_DIR=\"$HOME/"+args.archive+'"\n')
|
||||
if not foundwork:
|
||||
with open(os.environ['HOME']+'/.config/user-dirs.dirs', 'a') as fh:
|
||||
fh.write("XDG_WORK_DIR=\"$HOME/"+args.mountpoint+'"\n')
|
||||
|
||||
# start FUSE filesystem
|
||||
FUSE(WorkdirFS(args), os.path.join(os.environ['HOME'], args.mountpoint), nothreads=True, foreground=True)
|
||||
|
||||
|
|
@ -226,15 +324,31 @@ if __name__ == '__main__':
|
|||
#main(sys.argv[2], sys.argv[1])
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-a", "--archive",
|
||||
default='archive', help="Path to archivedir-base")
|
||||
default='archive/workdir', help="""Path to archivedir-base. When path
|
||||
starts with "/", it's an absolute path, else it is handled as path
|
||||
relative to users home.
|
||||
Defaults to »archive/workdir« """)
|
||||
parser.add_argument("-m", "--mountpoint",
|
||||
default='Work', help='Path to Workdir')
|
||||
parser.add_argument("-t", "--timeoffset", type=int, default=2, help="""If you're working
|
||||
default='Work', help="""Path to Workdir. This path is always
|
||||
relative to users homedir. "/" at the begin get removed.
|
||||
Defaults to »Work«""")
|
||||
parser.add_argument("-t", "--timeoffset", type=int, default=4, help="""If you're working
|
||||
all day till 3 o'clock in the morning, set it to 4, so next day
|
||||
archive-dir will be created 4 hours after midnight. You have 1h
|
||||
tolerance, if you're working one day a little bit longer""")
|
||||
parser.add_argument("-y", "--yearlydir", action="store_true")
|
||||
parser.add_argument("-M", "--monthlydir", action="store_true")
|
||||
tolerance, if you're working one day a little bit longer.
|
||||
Defaults to »4«""")
|
||||
parser.add_argument("-Y", "--yearlydir", action="store_true",
|
||||
help="""Create a yearly directory - named YYYY - under »archive«.
|
||||
Defaults to »False«""")
|
||||
parser.add_argument("-M", "--monthlydir", action="store_true",
|
||||
help="""Create a monthly directory - named MM - under »archive«.
|
||||
Defaults to »False«""")
|
||||
parser.add_argument("-W", "--weeklydir", action="store_true",
|
||||
help="""Create a weekly directory - named WW - under »archive«.
|
||||
Defaults to »False«""")
|
||||
parser.add_argument("-C", "--compress", action="store_false",
|
||||
help="""Compress each file in archive
|
||||
Defaults to »True«""")
|
||||
args = parser.parse_args()
|
||||
print(args)
|
||||
#root = os.environ['HOME']+'/archive'
|
||||
|
|
|
|||
Loading…
Reference in a new issue