【Azure Developer】使用 Python SDK连接Azure Storage Account, 计算Blob大小代码示例
- 2021 年 6 月 8 日
- 笔记
- 【Azure Developer】, Azure Developer, Azure Storage, Python 多线程代码, 使用 Python SDK连接Azure Storage Account, 计算Blob大小代码示例
问题描述
在微软云环境中，使用python SDK连接存储账号（Storage Account）需要计算Blob大小？虽然Azure提供了一个专用工具Azure Storage Explorer可以统计出Blob的大小：
但是它也是只能一个Blob Container一个的统计，如果Container数量巨大，这将是一个繁琐的工作。而作为开发者，应该让代码来帮助完成。下文使用最快上手的Python代码来计算Blob中容量的大小。
完整代码
import os, uuid, datetime, threading import logging from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, __version__ def calculateBlob(connect_string, count): try: blob_service_client = BlobServiceClient.from_connection_string(connect_string) except Exception as e: messages = str(count) + "Connect_String Error, Messages:" + e.args.__str__() print(messages) logging.info(messages) else: all_containers = blob_service_client.list_containers() for c in all_containers: count_name = c.name print(count_name) if count_name not in blobSize_Total: blobSize_Total[count_name] = 0 if count_name not in blobSize_Daily: blobSize_Daily[count_name] = 0 container_client = blob_service_client.get_container_client(count_name) generator = container_client.list_blobs() total_size_container = 0 daily_size_container = 0 for blob in generator: total_size_container += blob.size blob_create_time = blob.creation_time.strftime("%Y%m%d") if blob_create_time != now_date: continue else: # Calculate BlobSize in this month daily_size_container += blob.size # blobSize_Daily[count_name] += blob.size # /(1024*1024) # content_length - bytes blobSize_Total[count_name] += total_size_container / (1024 * 1024) blobSize_Daily[count_name] += daily_size_container / (1024 * 1024) return None if __name__ == '__main__': # connect string Connection_String_List ="DefaultEndpointsProtocol=https;AccountName=<storagename>;AccountKey=<key>;EndpointSuffix=core.chinacloudapi.cn" # for i in Connection_String: start = datetime.datetime.now() print(start) # 定义全局变量 - blobSize_Daily & blobSize_Total blobSize_Daily = {} blobSize_Total = {} now_date = datetime.datetime.now().strftime("%Y%m%d") print("开始计算") calculateBlob(Connection_String_List, 1) print("计算完成") print("统计当前新增大小") print(blobSize_Daily) print("统计Blob总大小") print(blobSize_Total) end = datetime.datetime.now() print(end)
如运行是没有Azure blob模块，可以使用 pip install azure-storage-blob 安装。以上代码运行结果如下：
如果有多个Storage Account，可以考虑加入多线程的方式来运行，在代码中增加一个myThread类，然后在 __main__ 中把 calculateBlob(Connection_String_List, 1) 运行替换为 many_thread(Connection_String_List) 即可。
class myThread(threading.Thread): def __init__(self, threadID, name, connection_string): threading.Thread.__init__(self) self.threadID = threadID self.name = name self.connection_string = connection_string def run(self): print("开始线程：" + self.name) calculateBlob(self.connection_string, self.threadID) print("退出线程：" + self.name) def many_thread(Connection_String_List): threads = [] for i in range(len(Connection_String_List)): # 循环创建32个线程 t = myThread(i, "Thread-" + str(i), Connection_String_List[i]) threads.append(t) for t in threads: # 循环启动32个线程 - 对应64个storage account t.start() for t in threads: t.join()
遇见问题
在多线程执行时，可能会遇见问题：(“Connection broken: ConnectionResetError(10054, ‘An existing connection was forcibly closed by the remote host’, None, 10054, None)”, ConnectionResetError(10054, ‘An existing connection was forcibly closed by the remote host’, None, 10054, None))，出现此问题大都是由于客户端使用了已经断开的连接导致所导致的。所以一定要仔细调试多线程关闭代码。是否是把还需要运行的线程给关闭了。导致了以上的错误消息。
附录一：多线程计算Blob的完整代码
import os, uuid, datetime, threading import logging from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, __version__ def calculateBlob(connect_string, count): try: blob_service_client = BlobServiceClient.from_connection_string(connect_string) except Exception as e: messages = str(count) + "Connect_String Error, Messages:" + e.args.__str__() print(messages) logging.info(messages) else: all_containers = blob_service_client.list_containers() for c in all_containers: count_name = c.name print(count_name) if count_name not in blobSize_Total: blobSize_Total[count_name] = 0 if count_name not in blobSize_Daily: blobSize_Daily[count_name] = 0 container_client = blob_service_client.get_container_client(count_name) generator = container_client.list_blobs() total_size_container = 0 daily_size_container = 0 for blob in generator: total_size_container += blob.size blob_create_time = blob.creation_time.strftime("%Y%m%d") if blob_create_time != now_date: continue else: # Calculate BlobSize in this month daily_size_container += blob.size # blobSize_Daily[count_name] += blob.size # /(1024*1024) # content_length - bytes blobSize_Total[count_name] += total_size_container / (1024 * 1024) blobSize_Daily[count_name] += daily_size_container / (1024 * 1024) return None class myThread(threading.Thread): def __init__(self, threadID, name, connection_string): threading.Thread.__init__(self) self.threadID = threadID self.name = name self.connection_string = connection_string def run(self): print("开始线程：" + self.name) calculateBlob(self.connection_string, self.threadID) print("退出线程：" + self.name) def many_thread(Connection_String_List): threads = [] for i in range(len(Connection_String_List)): # 循环创建32个线程 t = myThread(i, "Thread-" + str(i), Connection_String_List[i]) threads.append(t) for t in threads: # 循环启动32个线程 - 对应64个storage account t.start() for t in threads: t.join() if __name__ == '__main__': # connect string Connection_String_List = ['DefaultEndpointsProtocol=https;AccountName=<your storage account 1>;AccountKey=<Key 1>;EndpointSuffix=core.chinacloudapi.cn', 'DefaultEndpointsProtocol=https;AccountName=<your storage account 2>;AccountKey=<Key 2>;EndpointSuffix=core.chinacloudapi.cn'] # for i in Connection_String: start = datetime.datetime.now() print(start) # 定义全局变量 - blobSize_Daily & blobSize_Total blobSize_Daily = {} blobSize_Total = {} now_date = datetime.datetime.now().strftime("%Y%m%d") many_thread(Connection_String_List) print("Main Thread End") print(blobSize_Daily) print(blobSize_Total) end = datetime.datetime.now() print(end)
运行效果：
参考资料
快速入门：使用 Python v12 SDK 管理 blob ：//docs.azure.cn/zh-cn/storage/blobs/storage-quickstart-blobs-python
Python 列表(List) : //www.runoob.com/python/python-lists.html
BlobServiceClient Class : //docs.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.blobserviceclient?view=azure-python
