2023-10-24 16:44:52 +00:00
|
|
|
#!/usr/local/bin/python
|
2023-10-22 21:21:25 +00:00
|
|
|
|
|
|
|
import json
|
2023-10-24 17:28:45 +00:00
|
|
|
import logging
|
2023-10-22 21:21:25 +00:00
|
|
|
import pathlib
|
2023-10-24 17:28:45 +00:00
|
|
|
import sys
|
2023-10-22 21:21:25 +00:00
|
|
|
|
|
|
|
import minio
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
2023-10-24 17:28:45 +00:00
|
|
|
logger = logging.getLogger()
|
|
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
|
|
|
|
handler = logging.StreamHandler(sys.stdout)
|
|
|
|
handler.setLevel(logging.DEBUG)
|
|
|
|
formatter = logging.Formatter(
|
|
|
|
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
|
|
|
)
|
|
|
|
handler.setFormatter(formatter)
|
|
|
|
logger.addHandler(handler)
|
|
|
|
|
2023-10-22 21:21:25 +00:00
|
|
|
# Load the user configuration.
|
|
|
|
with open("/data/options.json", "r") as f:
|
|
|
|
config = json.load(f)
|
|
|
|
|
|
|
|
client = minio.Minio(
|
|
|
|
config["minio_url"],
|
|
|
|
access_key=config["minio_access_key"],
|
|
|
|
secret_key=config["minio_secret_key"],
|
|
|
|
)
|
2023-10-24 17:28:45 +00:00
|
|
|
logger.info("Created minio client")
|
2023-10-22 21:21:25 +00:00
|
|
|
|
|
|
|
found = client.bucket_exists(config["minio_bucket"])
|
|
|
|
if not found:
|
2023-10-24 17:33:12 +00:00
|
|
|
logger.warning("Creating bucket %s", config["minio_bucket"])
|
2023-10-22 21:21:25 +00:00
|
|
|
client.make_bucket(config["minio_bucket"])
|
|
|
|
else:
|
2023-10-24 17:28:45 +00:00
|
|
|
logger.info("Bucket %s already exists.", config["minio_bucket"])
|
2023-10-22 21:21:25 +00:00
|
|
|
|
|
|
|
backup_folder = pathlib.Path("/backup")
|
2023-10-24 17:33:12 +00:00
|
|
|
backup_files = list(backup_folder.iterdir())
|
2023-10-24 17:28:45 +00:00
|
|
|
logger.info("Found the folllowing files in /backup: %s", backup_files)
|
|
|
|
|
2023-10-24 17:50:17 +00:00
|
|
|
objects = client.list_objects(config["minio_bucket"])
|
|
|
|
object_names = [x.object_name for x in objects]
|
|
|
|
logger.info("Found the following files in s3: %s", object_names)
|
|
|
|
|
|
|
|
to_upload = []
|
|
|
|
for file in backup_files:
|
|
|
|
if file.name in object_names:
|
|
|
|
logger.info("File %s already exists in s3", file.name)
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
to_upload.append(file)
|
2023-10-22 21:21:25 +00:00
|
|
|
|
2023-10-24 17:33:12 +00:00
|
|
|
logger.warning(
|
2023-10-24 17:28:45 +00:00
|
|
|
"The following files do not already exist and will be backed up: %s", to_upload
|
|
|
|
)
|
2023-10-22 21:21:25 +00:00
|
|
|
|
|
|
|
for file in to_upload:
|
|
|
|
client.fput_object(
|
|
|
|
config["minio_bucket"],
|
|
|
|
file.name,
|
|
|
|
str(file.resolve()),
|
|
|
|
)
|
2023-10-24 17:33:12 +00:00
|
|
|
logger.warning("Uploaded %s", file.name)
|
2023-10-24 17:28:45 +00:00
|
|
|
|
|
|
|
logger.info("Done")
|
2023-10-22 21:21:25 +00:00
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
main()
|