|
|
|
@ -40,17 +40,17 @@ def get_file_identity(ctx_obj, file): |
|
|
|
return path |
|
|
|
|
|
|
|
|
|
|
|
def list_minio_dir(minio: Minio, bucket: str, prefix: str) -> List[str]: |
|
|
|
def list_s3_dir(s3: Minio, bucket: str, prefix: str) -> List[str]: |
|
|
|
found_files = [] |
|
|
|
for obj in minio.list_objects_v2(bucket, prefix=prefix): |
|
|
|
for obj in s3.list_objects_v2(bucket, prefix=prefix): |
|
|
|
if obj.is_dir: |
|
|
|
found_files.extend(list_minio_dir(minio, bucket, obj.object_name)) |
|
|
|
found_files.extend(list_s3_dir(s3, bucket, obj.object_name)) |
|
|
|
else: |
|
|
|
found_files.append(obj.object_name) |
|
|
|
return found_files |
|
|
|
|
|
|
|
|
|
|
|
def get_minio_client(config: any) -> Minio: |
|
|
|
def get_s3_client(config: any) -> Minio: |
|
|
|
host = config['host'] |
|
|
|
secure = config['secure'] |
|
|
|
access_key = config['access'] |
|
|
|
@ -90,20 +90,20 @@ def cli(ctx, debug, config, context, stdin, prefix): |
|
|
|
@click.option('--suffix', default=None) |
|
|
|
@click.pass_context |
|
|
|
def list_files(ctx, sha256sum, suffix): |
|
|
|
minio_config = ctx.obj['CONFIG']['minio'] |
|
|
|
minio_bucket = ctx.obj['CONTEXT'] |
|
|
|
s3_config = ctx.obj['CONFIG']['s3'] |
|
|
|
s3_bucket = ctx.obj['CONTEXT'] |
|
|
|
|
|
|
|
minio = get_minio_client(minio_config) |
|
|
|
s3 = get_s3_client(s3_config) |
|
|
|
|
|
|
|
if not minio.bucket_exists(minio_bucket): |
|
|
|
minio.make_bucket(minio_bucket) |
|
|
|
if not s3.bucket_exists(s3_bucket): |
|
|
|
s3.make_bucket(s3_bucket) |
|
|
|
|
|
|
|
found_files: List[str] = [] |
|
|
|
found_objects: List[str] = [] |
|
|
|
|
|
|
|
for obj in minio.list_objects_v2(minio_bucket, recursive=False): |
|
|
|
for obj in s3.list_objects_v2(s3_bucket, recursive=False): |
|
|
|
if obj.is_dir: |
|
|
|
found_objects.extend(list_minio_dir(minio, minio_bucket, obj.object_name)) |
|
|
|
found_objects.extend(list_s3_dir(s3, s3_bucket, obj.object_name)) |
|
|
|
else: |
|
|
|
found_objects.append(obj.object_name) |
|
|
|
|
|
|
|
@ -118,7 +118,7 @@ def list_files(ctx, sha256sum, suffix): |
|
|
|
file = file.strip() |
|
|
|
|
|
|
|
if sha256sum: |
|
|
|
stat = minio.stat_object(minio_bucket, obj) |
|
|
|
stat = s3.stat_object(s3_bucket, obj) |
|
|
|
sha256sum_value = stat.metadata[get_metadata_name("SHA256SUM")] |
|
|
|
file = f'{sha256sum_value} {file}' |
|
|
|
|
|
|
|
@ -127,17 +127,53 @@ def list_files(ctx, sha256sum, suffix): |
|
|
|
print(os.linesep.join(found_files)) |
|
|
|
|
|
|
|
|
|
|
|
@cli.command(name="match") |
|
|
|
@click.pass_context |
|
|
|
@click.argument('files', nargs=-1) |
|
|
|
def check_matched_files_hashes(ctx, files): |
|
|
|
s3_config = ctx.obj['CONFIG']['s3'] |
|
|
|
s3_bucket = ctx.obj['CONTEXT'] |
|
|
|
|
|
|
|
s3 = get_s3_client(s3_config) |
|
|
|
|
|
|
|
if not s3.bucket_exists(s3_bucket): |
|
|
|
s3.make_bucket(s3_bucket) |
|
|
|
|
|
|
|
changed_files: List[str] = [] |
|
|
|
|
|
|
|
if ctx.obj['READ_STDIN']: |
|
|
|
files = get_clean_stdin_iterator(click.get_text_stream('stdin')) |
|
|
|
|
|
|
|
for file in files: |
|
|
|
file_identity = f'{get_file_identity(ctx.obj, file)}.json' |
|
|
|
try: |
|
|
|
file_object = s3.stat_object(s3_bucket, file_identity) |
|
|
|
stored_file_hash = file_object.metadata[get_metadata_name("SHA256SUM")] |
|
|
|
sha256sum = hashlib.sha256() |
|
|
|
with open(file, 'rb') as f: |
|
|
|
for byte_block in iter(lambda: f.read(BUF_SIZE), b""): |
|
|
|
sha256sum.update(byte_block) |
|
|
|
calculated_file_hash = sha256sum.hexdigest() |
|
|
|
if calculated_file_hash == stored_file_hash: |
|
|
|
changed_files.append(file) |
|
|
|
except NoSuchKey as e: |
|
|
|
continue |
|
|
|
except ValueError or ResponseError as e: |
|
|
|
print(f'ERROR: {file} {e}') |
|
|
|
|
|
|
|
print(os.linesep.join(changed_files)) |
|
|
|
|
|
|
|
@cli.command(name="check") |
|
|
|
@click.pass_context |
|
|
|
@click.argument('files', nargs=-1) |
|
|
|
def check_changed_files_hashes(ctx, files): |
|
|
|
minio_config = ctx.obj['CONFIG']['minio'] |
|
|
|
minio_bucket = ctx.obj['CONTEXT'] |
|
|
|
s3_config = ctx.obj['CONFIG']['s3'] |
|
|
|
s3_bucket = ctx.obj['CONTEXT'] |
|
|
|
|
|
|
|
minio = get_minio_client(minio_config) |
|
|
|
s3 = get_s3_client(s3_config) |
|
|
|
|
|
|
|
if not minio.bucket_exists(minio_bucket): |
|
|
|
minio.make_bucket(minio_bucket) |
|
|
|
if not s3.bucket_exists(s3_bucket): |
|
|
|
s3.make_bucket(s3_bucket) |
|
|
|
|
|
|
|
changed_files: List[str] = [] |
|
|
|
|
|
|
|
@ -147,7 +183,7 @@ def check_changed_files_hashes(ctx, files): |
|
|
|
for file in files: |
|
|
|
file_identity = f'{get_file_identity(ctx.obj, file)}.json' |
|
|
|
try: |
|
|
|
file_object = minio.stat_object(minio_bucket, file_identity) |
|
|
|
file_object = s3.stat_object(s3_bucket, file_identity) |
|
|
|
stored_file_hash = file_object.metadata[get_metadata_name("SHA256SUM")] |
|
|
|
sha256sum = hashlib.sha256() |
|
|
|
with open(file, 'rb') as f: |
|
|
|
@ -168,13 +204,13 @@ def check_changed_files_hashes(ctx, files): |
|
|
|
@click.pass_context |
|
|
|
@click.argument('files', nargs=-1) |
|
|
|
def update_changed_files_hashes(ctx, files): |
|
|
|
minio_config = ctx.obj['CONFIG']['minio'] |
|
|
|
minio_bucket = ctx.obj['CONTEXT'] |
|
|
|
s3_config = ctx.obj['CONFIG']['s3'] |
|
|
|
s3_bucket = ctx.obj['CONTEXT'] |
|
|
|
|
|
|
|
minio = get_minio_client(minio_config) |
|
|
|
s3 = get_s3_client(s3_config) |
|
|
|
|
|
|
|
if not minio.bucket_exists(minio_bucket): |
|
|
|
minio.make_bucket(minio_bucket) |
|
|
|
if not s3.bucket_exists(s3_bucket): |
|
|
|
s3.make_bucket(s3_bucket) |
|
|
|
|
|
|
|
updated_files: List[str] = [] |
|
|
|
|
|
|
|
@ -196,8 +232,8 @@ def update_changed_files_hashes(ctx, files): |
|
|
|
data.seek(0, os.SEEK_END) |
|
|
|
data_length = data.tell() |
|
|
|
data.seek(0) |
|
|
|
minio.put_object( |
|
|
|
minio_bucket, |
|
|
|
s3.put_object( |
|
|
|
s3_bucket, |
|
|
|
file_identity, |
|
|
|
data, |
|
|
|
data_length, |
|
|
|
@ -217,13 +253,13 @@ def update_changed_files_hashes(ctx, files): |
|
|
|
@click.pass_context |
|
|
|
@click.argument('files', nargs=-1) |
|
|
|
def store_files(ctx, files): |
|
|
|
minio_config = ctx.obj['CONFIG']['minio'] |
|
|
|
minio_bucket = f'{ctx.obj["CONTEXT"]}-data' |
|
|
|
s3_config = ctx.obj['CONFIG']['s3'] |
|
|
|
s3_bucket = f'{ctx.obj["CONTEXT"]}-data' |
|
|
|
|
|
|
|
minio = get_minio_client(minio_config) |
|
|
|
s3 = get_s3_client(s3_config) |
|
|
|
|
|
|
|
if not minio.bucket_exists(minio_bucket): |
|
|
|
minio.make_bucket(minio_bucket) |
|
|
|
if not s3.bucket_exists(s3_bucket): |
|
|
|
s3.make_bucket(s3_bucket) |
|
|
|
|
|
|
|
stored_files: List[str] = [] |
|
|
|
|
|
|
|
@ -233,8 +269,8 @@ def store_files(ctx, files): |
|
|
|
for file in files: |
|
|
|
file_identity = get_file_identity(ctx.obj, file) |
|
|
|
try: |
|
|
|
minio.fput_object( |
|
|
|
minio_bucket, |
|
|
|
s3.fput_object( |
|
|
|
s3_bucket, |
|
|
|
file_identity, |
|
|
|
file, |
|
|
|
content_type="application/octet-stream" |
|
|
|
@ -251,13 +287,13 @@ def store_files(ctx, files): |
|
|
|
@click.option('-d', '--destination', default=None) |
|
|
|
@click.argument('files', nargs=-1) |
|
|
|
def retrieve_files(ctx, destination, files): |
|
|
|
minio_config = ctx.obj['CONFIG']['minio'] |
|
|
|
minio_bucket = f'{ctx.obj["CONTEXT"]}-data' |
|
|
|
s3_config = ctx.obj['CONFIG']['s3'] |
|
|
|
s3_bucket = f'{ctx.obj["CONTEXT"]}-data' |
|
|
|
|
|
|
|
minio = get_minio_client(minio_config) |
|
|
|
s3 = get_s3_client(s3_config) |
|
|
|
|
|
|
|
if not minio.bucket_exists(minio_bucket): |
|
|
|
minio.make_bucket(minio_bucket) |
|
|
|
if not s3.bucket_exists(s3_bucket): |
|
|
|
s3.make_bucket(s3_bucket) |
|
|
|
|
|
|
|
retrieved_files: List[str] = [] |
|
|
|
|
|
|
|
@ -270,8 +306,8 @@ def retrieve_files(ctx, destination, files): |
|
|
|
if destination is not None: |
|
|
|
file_destination = os.path.join(destination, file_identity) |
|
|
|
try: |
|
|
|
minio.fget_object( |
|
|
|
minio_bucket, |
|
|
|
s3.fget_object( |
|
|
|
s3_bucket, |
|
|
|
file_identity, |
|
|
|
file_destination |
|
|
|
) |
|
|
|
|
