diff --git a/.github/check_and_update.py b/.github/check_and_update.py index 704b658e67..41d041b7f7 100644 --- a/.github/check_and_update.py +++ b/.github/check_and_update.py @@ -1,85 +1,88 @@ -import hashlib -from huggingface_hub import hf_hub_download, get_paths_info -import requests -import sys -import os - -uri = sys.argv[1] -file_name = uri.split('/')[-1] - -# Function to parse the URI and determine download method -def parse_uri(uri): - if uri.startswith('huggingface://'): - repo_id = uri.split('://')[1] - return 'huggingface', repo_id.rsplit('/', 1)[0] - elif 'huggingface.co' in uri: - parts = uri.split('/resolve/') - if len(parts) > 1: - repo_path = parts[0].split('https://huggingface.co/')[-1] - return 'huggingface', repo_path - return 'direct', uri - -def calculate_sha256(file_path): - sha256_hash = hashlib.sha256() - with open(file_path, 'rb') as f: - for byte_block in iter(lambda: f.read(4096), b''): - sha256_hash.update(byte_block) - return sha256_hash.hexdigest() - -def manual_safety_check_hf(repo_id): - scanResponse = requests.get('https://huggingface.co/api/models/' + repo_id + "/scan") - scan = scanResponse.json() - # Check if 'hasUnsafeFile' exists in the response - if 'hasUnsafeFile' in scan: - if scan['hasUnsafeFile']: - return scan - else: - return None - else: - return None - -download_type, repo_id_or_url = parse_uri(uri) - -new_checksum = None -file_path = None - -# Decide download method based on URI type -if download_type == 'huggingface': - # Check if the repo is flagged as dangerous by HF - hazard = manual_safety_check_hf(repo_id_or_url) - if hazard != None: - print(f'Error: HuggingFace has detected security problems for {repo_id_or_url}: {str(hazard)}', filename=file_name) - sys.exit(5) - # Use HF API to pull sha - for file in get_paths_info(repo_id_or_url, [file_name], repo_type='model'): - try: - new_checksum = file.lfs.sha256 - break - except Exception as e: - print(f'Error from Hugging Face Hub: {str(e)}', file=sys.stderr) - sys.exit(2) - if new_checksum is None: - try: - file_path = hf_hub_download(repo_id=repo_id_or_url, filename=file_name) - except Exception as e: - print(f'Error from Hugging Face Hub: {str(e)}', file=sys.stderr) - sys.exit(2) -else: - response = requests.get(repo_id_or_url) - if response.status_code == 200: - with open(file_name, 'wb') as f: - f.write(response.content) - file_path = file_name - elif response.status_code == 404: - print(f'File not found: {response.status_code}', file=sys.stderr) - sys.exit(2) - else: - print(f'Error downloading file: {response.status_code}', file=sys.stderr) - sys.exit(1) - -if new_checksum is None: - new_checksum = calculate_sha256(file_path) - print(new_checksum) - os.remove(file_path) -else: - print(new_checksum) +import hashlib +from huggingface_hub import hf_hub_download, get_paths_info +import requests +import sys +import os + +uri = sys.argv[1] +file_name = uri.split('/')[-1] + +# Function to parse the URI and determine download method. +# A URI that pins a revision (.../resolve//) downloads that +# revision, so its checksum has to come from the same revision, not from main. +def parse_uri(uri): + if uri.startswith('huggingface://'): + repo_id = uri.split('://')[1] + return 'huggingface', repo_id.rsplit('/', 1)[0], None + elif 'huggingface.co' in uri: + parts = uri.split('/resolve/') + if len(parts) > 1: + repo_path = parts[0].split('https://huggingface.co/')[-1] + revision = parts[1].split('/', 1)[0] + return 'huggingface', repo_path, revision + return 'direct', uri, None + +def calculate_sha256(file_path): + sha256_hash = hashlib.sha256() + with open(file_path, 'rb') as f: + for byte_block in iter(lambda: f.read(4096), b''): + sha256_hash.update(byte_block) + return sha256_hash.hexdigest() + +def manual_safety_check_hf(repo_id): + scanResponse = requests.get('https://huggingface.co/api/models/' + repo_id + "/scan") + scan = scanResponse.json() + # Check if 'hasUnsafeFile' exists in the response + if 'hasUnsafeFile' in scan: + if scan['hasUnsafeFile']: + return scan + else: + return None + else: + return None + +download_type, repo_id_or_url, revision = parse_uri(uri) + +new_checksum = None +file_path = None + +# Decide download method based on URI type +if download_type == 'huggingface': + # Check if the repo is flagged as dangerous by HF + hazard = manual_safety_check_hf(repo_id_or_url) + if hazard != None: + print(f'Error: HuggingFace has detected security problems for {repo_id_or_url}: {str(hazard)}', filename=file_name) + sys.exit(5) + # Use HF API to pull sha + for file in get_paths_info(repo_id_or_url, [file_name], repo_type='model', revision=revision): + try: + new_checksum = file.lfs.sha256 + break + except Exception as e: + print(f'Error from Hugging Face Hub: {str(e)}', file=sys.stderr) + sys.exit(2) + if new_checksum is None: + try: + file_path = hf_hub_download(repo_id=repo_id_or_url, filename=file_name, revision=revision) + except Exception as e: + print(f'Error from Hugging Face Hub: {str(e)}', file=sys.stderr) + sys.exit(2) +else: + response = requests.get(repo_id_or_url) + if response.status_code == 200: + with open(file_name, 'wb') as f: + f.write(response.content) + file_path = file_name + elif response.status_code == 404: + print(f'File not found: {response.status_code}', file=sys.stderr) + sys.exit(2) + else: + print(f'Error downloading file: {response.status_code}', file=sys.stderr) + sys.exit(1) + +if new_checksum is None: + new_checksum = calculate_sha256(file_path) + print(new_checksum) + os.remove(file_path) +else: + print(new_checksum) diff --git a/gallery/index.yaml b/gallery/index.yaml index 3ef48ea691..6cea0e0baa 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -31977,7 +31977,7 @@ sha256: 9a773289bd321ed879cd3a76545f6aca506e403851a45fb315420aee45625be2 uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q4_K_M.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf overrides: backend: llama-cpp @@ -32030,7 +32030,7 @@ sha256: d4e36902ba8f0ee7b333596030a59d650f88ac76660b4d9ba5ff77c3c2258fc0 uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q8_0.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf overrides: backend: llama-cpp @@ -32085,7 +32085,7 @@ sha256: 9a773289bd321ed879cd3a76545f6aca506e403851a45fb315420aee45625be2 uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q4_K_M.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf overrides: backend: llama-cpp @@ -32141,7 +32141,7 @@ sha256: d4e36902ba8f0ee7b333596030a59d650f88ac76660b4d9ba5ff77c3c2258fc0 uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q8_0.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf overrides: backend: llama-cpp @@ -32196,7 +32196,7 @@ sha256: 9a773289bd321ed879cd3a76545f6aca506e403851a45fb315420aee45625be2 uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q4_K_M.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf overrides: backend: llama-cpp @@ -32251,7 +32251,7 @@ sha256: d4e36902ba8f0ee7b333596030a59d650f88ac76660b4d9ba5ff77c3c2258fc0 uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q8_0.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf overrides: backend: llama-cpp @@ -32307,7 +32307,7 @@ sha256: 9a773289bd321ed879cd3a76545f6aca506e403851a45fb315420aee45625be2 uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q4_K_M.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf overrides: backend: llama-cpp @@ -32364,7 +32364,7 @@ sha256: d4e36902ba8f0ee7b333596030a59d650f88ac76660b4d9ba5ff77c3c2258fc0 uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q8_0.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf overrides: backend: llama-cpp diff --git a/gallery/models/index.yaml b/gallery/models/index.yaml index 1744f0c62c..34f3d6b32d 100644 --- a/gallery/models/index.yaml +++ b/gallery/models/index.yaml @@ -37595,7 +37595,7 @@ uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q4_K_M.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e overrides: backend: llama-cpp context_size: 8192 @@ -37656,7 +37656,7 @@ uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q8_0.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e overrides: backend: llama-cpp context_size: 8192 @@ -37719,7 +37719,7 @@ uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q4_K_M.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e overrides: backend: llama-cpp context_size: 8192 @@ -37783,7 +37783,7 @@ uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q8_0.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e overrides: backend: llama-cpp context_size: 8192 @@ -37846,7 +37846,7 @@ uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q4_K_M.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e overrides: backend: llama-cpp context_size: 8192 @@ -37909,7 +37909,7 @@ uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q8_0.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e overrides: backend: llama-cpp context_size: 8192 @@ -37973,7 +37973,7 @@ uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q4_K_M.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e overrides: backend: llama-cpp context_size: 8192 @@ -38038,7 +38038,7 @@ uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/LFM3-Audio-2B-Q8_0.gguf - filename: mmproj-LFM3-Audio-2B-Q8_0.gguf uri: https://huggingface.co/LiquidAI/LFM3-Audio-2B-GGUF/resolve/fcfae9027c00a6f3f872c48a841cfeb77e2b5a1c/mmproj-LFM3-Audio-2B-Q8_0.gguf - sha256: 2b6648b1fd48af16aa0877b0592b144d15070783673e038a379fdbc51d40e365 + sha256: 26601e904245a85bd8f3cf320a629e897cfc01c1d09369e8b4fd951f13024d3e overrides: backend: llama-cpp context_size: 8192