From 9afa0bbc3c2613a2860a5e85ef2a29b58a24605e Mon Sep 17 00:00:00 2001 From: Drew Short Date: Tue, 14 Apr 2020 21:19:06 -0500 Subject: [PATCH] Added a match function --- acm.py | 114 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 39 deletions(-) diff --git a/acm.py b/acm.py index 83fa39f..bbacc47 100644 --- a/acm.py +++ b/acm.py @@ -40,17 +40,17 @@ def get_file_identity(ctx_obj, file): return path -def list_minio_dir(minio: Minio, bucket: str, prefix: str) -> List[str]: +def list_s3_dir(s3: Minio, bucket: str, prefix: str) -> List[str]: found_files = [] - for obj in minio.list_objects_v2(bucket, prefix=prefix): + for obj in s3.list_objects_v2(bucket, prefix=prefix): if obj.is_dir: - found_files.extend(list_minio_dir(minio, bucket, obj.object_name)) + found_files.extend(list_s3_dir(s3, bucket, obj.object_name)) else: found_files.append(obj.object_name) return found_files -def get_minio_client(config: any) -> Minio: +def get_s3_client(config: any) -> Minio: host = config['host'] secure = config['secure'] access_key = config['access'] @@ -90,20 +90,20 @@ def cli(ctx, debug, config, context, stdin, prefix): @click.option('--suffix', default=None) @click.pass_context def list_files(ctx, sha256sum, suffix): - minio_config = ctx.obj['CONFIG']['minio'] - minio_bucket = ctx.obj['CONTEXT'] + s3_config = ctx.obj['CONFIG']['s3'] + s3_bucket = ctx.obj['CONTEXT'] - minio = get_minio_client(minio_config) + s3 = get_s3_client(s3_config) - if not minio.bucket_exists(minio_bucket): - minio.make_bucket(minio_bucket) + if not s3.bucket_exists(s3_bucket): + s3.make_bucket(s3_bucket) found_files: List[str] = [] found_objects: List[str] = [] - for obj in minio.list_objects_v2(minio_bucket, recursive=False): + for obj in s3.list_objects_v2(s3_bucket, recursive=False): if obj.is_dir: - found_objects.extend(list_minio_dir(minio, minio_bucket, obj.object_name)) + found_objects.extend(list_s3_dir(s3, s3_bucket, obj.object_name)) else: found_objects.append(obj.object_name) @@ -118,7 +118,7 @@ def list_files(ctx, sha256sum, suffix): file = file.strip() if sha256sum: - stat = minio.stat_object(minio_bucket, obj) + stat = s3.stat_object(s3_bucket, obj) sha256sum_value = stat.metadata[get_metadata_name("SHA256SUM")] file = f'{sha256sum_value} {file}' @@ -127,17 +127,53 @@ def list_files(ctx, sha256sum, suffix): print(os.linesep.join(found_files)) +@cli.command(name="match") +@click.pass_context +@click.argument('files', nargs=-1) +def check_matched_files_hashes(ctx, files): + s3_config = ctx.obj['CONFIG']['s3'] + s3_bucket = ctx.obj['CONTEXT'] + + s3 = get_s3_client(s3_config) + + if not s3.bucket_exists(s3_bucket): + s3.make_bucket(s3_bucket) + + changed_files: List[str] = [] + + if ctx.obj['READ_STDIN']: + files = get_clean_stdin_iterator(click.get_text_stream('stdin')) + + for file in files: + file_identity = f'{get_file_identity(ctx.obj, file)}.json' + try: + file_object = s3.stat_object(s3_bucket, file_identity) + stored_file_hash = file_object.metadata[get_metadata_name("SHA256SUM")] + sha256sum = hashlib.sha256() + with open(file, 'rb') as f: + for byte_block in iter(lambda: f.read(BUF_SIZE), b""): + sha256sum.update(byte_block) + calculated_file_hash = sha256sum.hexdigest() + if calculated_file_hash == stored_file_hash: + changed_files.append(file) + except NoSuchKey as e: + continue + except ValueError or ResponseError as e: + print(f'ERROR: {file} {e}') + + print(os.linesep.join(changed_files)) + @cli.command(name="check") @click.pass_context @click.argument('files', nargs=-1) def check_changed_files_hashes(ctx, files): - minio_config = ctx.obj['CONFIG']['minio'] - minio_bucket = ctx.obj['CONTEXT'] + s3_config = ctx.obj['CONFIG']['s3'] + s3_bucket = ctx.obj['CONTEXT'] - minio = get_minio_client(minio_config) + s3 = get_s3_client(s3_config) - if not minio.bucket_exists(minio_bucket): - minio.make_bucket(minio_bucket) + if not s3.bucket_exists(s3_bucket): + s3.make_bucket(s3_bucket) changed_files: List[str] = [] @@ -147,7 +183,7 @@ def check_changed_files_hashes(ctx, files): for file in files: file_identity = f'{get_file_identity(ctx.obj, file)}.json' try: - file_object = minio.stat_object(minio_bucket, file_identity) + file_object = s3.stat_object(s3_bucket, file_identity) stored_file_hash = file_object.metadata[get_metadata_name("SHA256SUM")] sha256sum = hashlib.sha256() with open(file, 'rb') as f: @@ -168,13 +204,13 @@ def check_changed_files_hashes(ctx, files): @click.pass_context @click.argument('files', nargs=-1) def update_changed_files_hashes(ctx, files): - minio_config = ctx.obj['CONFIG']['minio'] - minio_bucket = ctx.obj['CONTEXT'] + s3_config = ctx.obj['CONFIG']['s3'] + s3_bucket = ctx.obj['CONTEXT'] - minio = get_minio_client(minio_config) + s3 = get_s3_client(s3_config) - if not minio.bucket_exists(minio_bucket): - minio.make_bucket(minio_bucket) + if not s3.bucket_exists(s3_bucket): + s3.make_bucket(s3_bucket) updated_files: List[str] = [] @@ -196,8 +232,8 @@ def update_changed_files_hashes(ctx, files): data.seek(0, os.SEEK_END) data_length = data.tell() data.seek(0) - minio.put_object( - minio_bucket, + s3.put_object( + s3_bucket, file_identity, data, data_length, @@ -217,13 +253,13 @@ def update_changed_files_hashes(ctx, files): @click.pass_context @click.argument('files', nargs=-1) def store_files(ctx, files): - minio_config = ctx.obj['CONFIG']['minio'] - minio_bucket = f'{ctx.obj["CONTEXT"]}-data' + s3_config = ctx.obj['CONFIG']['s3'] + s3_bucket = f'{ctx.obj["CONTEXT"]}-data' - minio = get_minio_client(minio_config) + s3 = get_s3_client(s3_config) - if not minio.bucket_exists(minio_bucket): - minio.make_bucket(minio_bucket) + if not s3.bucket_exists(s3_bucket): + s3.make_bucket(s3_bucket) stored_files: List[str] = [] @@ -233,8 +269,8 @@ def store_files(ctx, files): for file in files: file_identity = get_file_identity(ctx.obj, file) try: - minio.fput_object( - minio_bucket, + s3.fput_object( + s3_bucket, file_identity, file, content_type="application/octet-stream" @@ -251,13 +287,13 @@ def store_files(ctx, files): @click.option('-d', '--destination', default=None) @click.argument('files', nargs=-1) def retrieve_files(ctx, destination, files): - minio_config = ctx.obj['CONFIG']['minio'] - minio_bucket = f'{ctx.obj["CONTEXT"]}-data' + s3_config = ctx.obj['CONFIG']['s3'] + s3_bucket = f'{ctx.obj["CONTEXT"]}-data' - minio = get_minio_client(minio_config) + s3 = get_s3_client(s3_config) - if not minio.bucket_exists(minio_bucket): - minio.make_bucket(minio_bucket) + if not s3.bucket_exists(s3_bucket): + s3.make_bucket(s3_bucket) retrieved_files: List[str] = [] @@ -270,8 +306,8 @@ def retrieve_files(ctx, destination, files): if destination is not None: file_destination = os.path.join(destination, file_identity) try: - minio.fget_object( - minio_bucket, + s3.fget_object( + s3_bucket, file_identity, file_destination )