【Azure Developer】使用 Python SDK連接Azure Storage Account, 計算Blob大小程式碼示例
- 2021 年 6 月 8 日
- 筆記
- 【Azure Developer】, Azure Developer, Azure Storage, Python 多執行緒程式碼, 使用 Python SDK連接Azure Storage Account, 計算Blob大小程式碼示例
問題描述
在微軟雲環境中，使用python SDK連接存儲帳號（Storage Account）需要計算Blob大小？雖然Azure提供了一個專用工具Azure Storage Explorer可以統計出Blob的大小：
但是它也是只能一個Blob Container一個的統計，如果Container數量巨大，這將是一個繁瑣的工作。而作為開發者，應該讓程式碼來幫助完成。下文使用最快上手的Python程式碼來計算Blob中容量的大小。
完整程式碼
import os, uuid, datetime, threading import logging from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, __version__ def calculateBlob(connect_string, count): try: blob_service_client = BlobServiceClient.from_connection_string(connect_string) except Exception as e: messages = str(count) + "Connect_String Error, Messages:" + e.args.__str__() print(messages) logging.info(messages) else: all_containers = blob_service_client.list_containers() for c in all_containers: count_name = c.name print(count_name) if count_name not in blobSize_Total: blobSize_Total[count_name] = 0 if count_name not in blobSize_Daily: blobSize_Daily[count_name] = 0 container_client = blob_service_client.get_container_client(count_name) generator = container_client.list_blobs() total_size_container = 0 daily_size_container = 0 for blob in generator: total_size_container += blob.size blob_create_time = blob.creation_time.strftime("%Y%m%d") if blob_create_time != now_date: continue else: # Calculate BlobSize in this month daily_size_container += blob.size # blobSize_Daily[count_name] += blob.size # /(1024*1024) # content_length - bytes blobSize_Total[count_name] += total_size_container / (1024 * 1024) blobSize_Daily[count_name] += daily_size_container / (1024 * 1024) return None if __name__ == '__main__': # connect string Connection_String_List ="DefaultEndpointsProtocol=https;AccountName=<storagename>;AccountKey=<key>;EndpointSuffix=core.chinacloudapi.cn" # for i in Connection_String: start = datetime.datetime.now() print(start) # 定義全局變數 - blobSize_Daily & blobSize_Total blobSize_Daily = {} blobSize_Total = {} now_date = datetime.datetime.now().strftime("%Y%m%d") print("開始計算") calculateBlob(Connection_String_List, 1) print("計算完成") print("統計當前新增大小") print(blobSize_Daily) print("統計Blob總大小") print(blobSize_Total) end = datetime.datetime.now() print(end)
如運行是沒有Azure blob模組，可以使用 pip install azure-storage-blob 安裝。以上程式碼運行結果如下：
如果有多個Storage Account，可以考慮加入多執行緒的方式來運行，在程式碼中增加一個myThread類，然後在 __main__ 中把 calculateBlob(Connection_String_List, 1) 運行替換為 many_thread(Connection_String_List) 即可。
class myThread(threading.Thread): def __init__(self, threadID, name, connection_string): threading.Thread.__init__(self) self.threadID = threadID self.name = name self.connection_string = connection_string def run(self): print("開始執行緒：" + self.name) calculateBlob(self.connection_string, self.threadID) print("退出執行緒：" + self.name) def many_thread(Connection_String_List): threads = [] for i in range(len(Connection_String_List)): # 循環創建32個執行緒 t = myThread(i, "Thread-" + str(i), Connection_String_List[i]) threads.append(t) for t in threads: # 循環啟動32個執行緒 - 對應64個storage account t.start() for t in threads: t.join()
遇見問題
在多執行緒執行時，可能會遇見問題：(“Connection broken: ConnectionResetError(10054, ‘An existing connection was forcibly closed by the remote host’, None, 10054, None)”, ConnectionResetError(10054, ‘An existing connection was forcibly closed by the remote host’, None, 10054, None))，出現此問題大都是由於客戶端使用了已經斷開的連接導致所導致的。所以一定要仔細調試多執行緒關閉程式碼。是否是把還需要運行的執行緒給關閉了。導致了以上的錯誤消息。
附錄一：多執行緒計算Blob的完整程式碼
import os, uuid, datetime, threading import logging from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, __version__ def calculateBlob(connect_string, count): try: blob_service_client = BlobServiceClient.from_connection_string(connect_string) except Exception as e: messages = str(count) + "Connect_String Error, Messages:" + e.args.__str__() print(messages) logging.info(messages) else: all_containers = blob_service_client.list_containers() for c in all_containers: count_name = c.name print(count_name) if count_name not in blobSize_Total: blobSize_Total[count_name] = 0 if count_name not in blobSize_Daily: blobSize_Daily[count_name] = 0 container_client = blob_service_client.get_container_client(count_name) generator = container_client.list_blobs() total_size_container = 0 daily_size_container = 0 for blob in generator: total_size_container += blob.size blob_create_time = blob.creation_time.strftime("%Y%m%d") if blob_create_time != now_date: continue else: # Calculate BlobSize in this month daily_size_container += blob.size # blobSize_Daily[count_name] += blob.size # /(1024*1024) # content_length - bytes blobSize_Total[count_name] += total_size_container / (1024 * 1024) blobSize_Daily[count_name] += daily_size_container / (1024 * 1024) return None class myThread(threading.Thread): def __init__(self, threadID, name, connection_string): threading.Thread.__init__(self) self.threadID = threadID self.name = name self.connection_string = connection_string def run(self): print("開始執行緒：" + self.name) calculateBlob(self.connection_string, self.threadID) print("退出執行緒：" + self.name) def many_thread(Connection_String_List): threads = [] for i in range(len(Connection_String_List)): # 循環創建32個執行緒 t = myThread(i, "Thread-" + str(i), Connection_String_List[i]) threads.append(t) for t in threads: # 循環啟動32個執行緒 - 對應64個storage account t.start() for t in threads: t.join() if __name__ == '__main__': # connect string Connection_String_List = ['DefaultEndpointsProtocol=https;AccountName=<your storage account 1>;AccountKey=<Key 1>;EndpointSuffix=core.chinacloudapi.cn', 'DefaultEndpointsProtocol=https;AccountName=<your storage account 2>;AccountKey=<Key 2>;EndpointSuffix=core.chinacloudapi.cn'] # for i in Connection_String: start = datetime.datetime.now() print(start) # 定義全局變數 - blobSize_Daily & blobSize_Total blobSize_Daily = {} blobSize_Total = {} now_date = datetime.datetime.now().strftime("%Y%m%d") many_thread(Connection_String_List) print("Main Thread End") print(blobSize_Daily) print(blobSize_Total) end = datetime.datetime.now() print(end)
運行效果：
參考資料
快速入門：使用 Python v12 SDK 管理 blob ：//docs.azure.cn/zh-cn/storage/blobs/storage-quickstart-blobs-python
Python 列表(List) : //www.runoob.com/python/python-lists.html
BlobServiceClient Class : //docs.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.blobserviceclient?view=azure-python
