FormData/Go分片/分块文件上传
- 2020 年 3 月 26 日
- 筆記
FormData 接口提供了一种表示表单数据的键值对的构造方式，经过它的数据可以使用 XMLHttpRequest.send() 方法送出，本接口和此方法都相当简单直接。如果送出时的编码类型被设为 "multipart/form-data"，它会使用和表单一样的格式。
如果你想构建一个简单的GET请求，并且通过<form>的形式带有查询参数，可以将它直接传递给URLSearchParams。
更多解释MDN: https://developer.mozilla.org/zh-CN/docs/Web/API/FormData
分块(分片，统称分块了，确实只是发送一块数据)文件上传主要分2部分。
1. 前端js用file.slice可以从文件中切出一块一块的数据，然后用FormData包装一下，用XMLHttpRequest把切出来的数据块，一块一块send到server.
2. Server接收到的每一块都是一个multipart/form-data Form表单。可以在表单里放很多附属信息，文件名，大小，块大小，块索引，最总带上这块切出来的二进制数据。
multipart/form-data 数据
POST /upload HTTP/1.1 Host: localhost:8080 Content-Length: 2098072 User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36 Content-Type: multipart/form-data; boundary=----WebKitFormBoundarymtng0xrR3ASR7wx7 ------WebKitFormBoundaryHdBeczaB5xBq6d55 Content-Disposition: form-data; name="file_name" apache-maven-3.6.3-bin.zip ------WebKitFormBoundaryHdBeczaB5xBq6d55 Content-Disposition: form-data; name="file_size" 9602303 ------WebKitFormBoundaryHdBeczaB5xBq6d55 Content-Disposition: form-data; name="block_size" 2097152 ------WebKitFormBoundaryHdBeczaB5xBq6d55 Content-Disposition: form-data; name="total_blocks" 5 ------WebKitFormBoundaryHdBeczaB5xBq6d55 Content-Disposition: form-data; name="break_error" true ------WebKitFormBoundaryHdBeczaB5xBq6d55 Content-Disposition: form-data; name="index" 3 ------WebKitFormBoundaryHdBeczaB5xBq6d55 Content-Disposition: form-data; name="data"; filename="blob" Content-Type: application/octet-stream (binary)
在Server存储文件，基本也就2种方案：
1. 直接创建一个对应大小的文件，按照每块数据的offset位置，写进去。
2. 每个传过来的数据块，保存成一个单独的数据块文件，最后把所有文件块合并成文件。
我这里只是做了一份简单的演示代码，基本上是不能用于生产环境的。
Index.html，直接把js写进去了
1 <!DOCTYPE html> 2 <html> 3 <head> 4 <meta charset="utf8"> 5 <title>Multil-Blocks upload</title> 6 </head> 7 8 <body> 9 <h2>Multil-Blocks upload</h2> 10 11 <input id="file" type="file" /> 12 13 <input type="checkbox" id="multil_block_file">multil block file</input> 14 <button type="button" onclick="on_block_upload()">Block upload</button> 15 <button type="button" onclick="on_concurrency_upload()">Concurrency upload</button> 16 <hr/> 17 18 <div> 19 <label>File name: </label><span id="file_name"></span> 20 </div> 21 <div> 22 <label>File size: </label><span id="file_size"></span> 23 </div> 24 <div> 25 <label>Split blocks: </label><span id="block_count"></span> 26 </div> 27 28 <hr/> 29 30 <p id="upload_info"></p> 31 32 <script> 33 var Block_Size = 1024 * 1024 * 2; 34 35 var el_file = document.getElementById('file'); 36 var el_multil_block_file = document.getElementById('multil_block_file'); 37 var el_file_name = document.getElementById('file_name'); 38 var el_file_size = document.getElementById('file_size'); 39 var el_block_count = document.getElementById('block_count'); 40 var el_upload_info = document.getElementById('upload_info'); 41 42 var file = null; 43 var total_blocks = 0; 44 var block_index = -1; 45 var block_index_random_arr = []; 46 var form_data = null; 47 48 49 el_file.onchange = function() { 50 if (this.files.length === 0) return; 51 52 file = this.files[0]; 53 total_blocks = Math.ceil( file.size / Block_Size ); 54 55 el_file_name.innerText = file.name; 56 el_file_size.innerText = file.size; 57 el_block_count.innerText = total_blocks; 58 } 59 60 function print_info(msg) { 61 el_upload_info.innerHTML += `${msg}<br/>`; 62 } 63 64 function done() { 65 file = null; 66 total_blocks = 0; 67 block_index = -1; 68 form_data = null; 69 70 el_file.value = ''; 71 } 72 73 74 function get_base_form_data() { 75 var base_data = new FormData(); 76 base_data.append('file_name', file.name); 77 base_data.append('file_size', file.size); 78 base_data.append('block_size', Block_Size); 79 base_data.append('total_blocks', total_blocks); 80 base_data.append('break_error', true); 81 base_data.append('index', 0); 82 base_data.append('data', null); 83 84 return base_data 85 } 86 87 88 function build_block_index_random_arr() { 89 block_index_random_arr = new Array(total_blocks).fill(0).map((v,i) => i); 90 block_index_random_arr.sort((n, m) => Math.random() > .5 ? -1 : 1); 91 92 print_info(`Upload sequence: ${block_index_random_arr}`); 93 } 94 95 96 function post(index, success_cb, failed_cb) { 97 if (!form_data) { 98 form_data = get_base_form_data(); 99 } 100 var start = index * Block_Size; 101 var end = Math.min(file.size, start + Block_Size); 102 103 form_data.set('index', index); 104 form_data.set('data', file.slice(start, end)); 105 106 print_info(`Post ${index}/${total_blocks}, offset: ${start} -- ${end}`); 107 108 109 var xhr = new XMLHttpRequest(); 110 xhr.open('POST', '/upload', true); 111 /* 112 Browser-based general content types 113 Content-Type: multipart/form-data; boundary=----WebKitFormBoundarysXH5DIES2XFMuLXL 114 115 Error content type: 116 xhr.setRequestHeader('Content-Type', 'multipart/form-data'); 117 Content-Type: multipart/form-data; 118 */ 119 xhr.onreadystatechange = function() { 120 121 if (xhr.readyState === XMLHttpRequest.DONE) { 122 123 if (xhr.status >= 200 && xhr.status < 300 && success_cb) { 124 return success_cb(); 125 } 126 127 if (xhr.status >= 400 && failed_cb) { 128 failed_cb(); 129 } 130 } 131 } 132 133 // xhr.onerror event 134 xhr.send(form_data); 135 } 136 137 138 function block_upload() { 139 if (!file) { 140 return; 141 } 142 if (block_index + 1 >= total_blocks) { 143 return done(); 144 } 145 146 block_index++; 147 var index = block_index_random_arr[block_index]; 148 149 post(index, block_upload); 150 } 151 152 153 function concurrency_upload() { 154 if (!file || total_blocks === 0) { 155 return; 156 } 157 158 build_block_index_random_arr(); 159 160 form_data = get_base_form_data(); 161 form_data.set('break_error', false); 162 form_data.set('multil_block', el_multil_block_file.checked); 163 164 for (var i of block_index_random_arr) { 165 ((idx) => { 166 post(idx, null, function() { 167 print_info(`Failed: ${idx}`); 168 setTimeout(() => post(idx), 1000); 169 }); 170 })(i); 171 } 172 } 173 174 175 function on_block_upload() { 176 if (file) { 177 print_info('Block upload'); 178 179 form_data = get_base_form_data(); 180 form_data.set('multil_block', el_multil_block_file.checked); 181 182 build_block_index_random_arr(); 183 184 block_index = -1; 185 block_upload(); 186 } 187 } 188 189 function on_concurrency_upload() { 190 if (file) { 191 print_info('Concurrency upload'); 192 concurrency_upload(); 193 } 194 } 195 </script> 196 197 </body> 198 </html>
View Code
简单的Go server和保存文件，基本忽略所有的错误处理
1 package main 2 3 import ( 4 "fmt" 5 "io/ioutil" 6 "log" 7 "net/http" 8 "os" 9 "path" 10 "path/filepath" 11 "regexp" 12 "strconv" 13 "strings" 14 "syscall" 15 "text/template" 16 ) 17 18 type MultilBlockFile struct { 19 FileName string 20 Size int64 21 BlockSize int64 22 TotalBlocks int 23 Index int 24 Bufs []byte 25 BreakError bool 26 } 27 28 func fileIsExist(f string) bool { 29 _, err := os.Stat(f) 30 return err == nil || os.IsExist(err) 31 } 32 33 func lockFile(f *os.File) error { 34 err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) 35 if err != nil { 36 return fmt.Errorf("get flock failed. err: %s", err) 37 } 38 39 return nil 40 } 41 42 func unlockFile(f *os.File) error { 43 defer f.Close() 44 return syscall.Flock(int(f.Fd()), syscall.LOCK_UN) 45 } 46 47 func singleFileSave(mbf *MultilBlockFile) error { 48 49 dir, _ := filepath.Abs(filepath.Dir(os.Args[0])) 50 filePath := path.Join(dir, "tmp", mbf.FileName) 51 52 offset := int64(mbf.Index) * mbf.BlockSize 53 54 fmt.Println(">>> Single file save ---------------------") 55 fmt.Printf("Save file: %s n", filePath) 56 fmt.Printf("File offset: %d n", offset) 57 58 var f *os.File 59 var needTruncate bool = false 60 if !fileIsExist(filePath) { 61 needTruncate = true 62 } 63 64 f, _ = os.OpenFile(filePath, syscall.O_CREAT|syscall.O_WRONLY, 0777) 65 66 err := lockFile(f) 67 if err != nil { 68 if mbf.BreakError { 69 log.Fatalf("get flock failed. err: %s", err) 70 } else { 71 return err 72 } 73 } 74 75 if needTruncate { 76 f.Truncate(mbf.Size) 77 } 78 79 f.WriteAt(mbf.Bufs, offset) 80 81 unlockFile(f) 82 83 return nil 84 } 85 86 func multilBlocksSave(mbf *MultilBlockFile) error { 87 dir, _ := filepath.Abs(filepath.Dir(os.Args[0])) 88 tmpFolderPath := path.Join(dir, "tmp") 89 tmpFileName := fmt.Sprintf("%s.%d", mbf.FileName, mbf.Index) 90 fileBlockPath := path.Join(tmpFolderPath, tmpFileName) 91 92 f, _ := os.OpenFile(fileBlockPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_TRUNC, 0777) 93 defer f.Close() 94 95 f.Write(mbf.Bufs) 96 f.Close() 97 98 re := regexp.MustCompile(`(?i:^` + mbf.FileName + `).d$`) 99 100 files, _ := ioutil.ReadDir(tmpFolderPath) 101 matchFiles := make(map[string]bool) 102 103 for _, file := range files { 104 if file.IsDir() { 105 continue 106 } 107 108 fname := file.Name() 109 if re.MatchString(fname) { 110 matchFiles[fname] = true 111 } 112 } 113 114 if len(matchFiles) >= mbf.TotalBlocks { 115 lastFile, _ := os.OpenFile(path.Join(tmpFolderPath, mbf.FileName), syscall.O_CREAT|syscall.O_WRONLY, 0777) 116 lockFile(lastFile) 117 118 lastFile.Truncate(mbf.Size) 119 120 for name := range matchFiles { 121 tmpPath := path.Join(tmpFolderPath, name) 122 123 idxStr := name[strings.LastIndex(name, ".")+1:] 124 idx, _ := strconv.ParseInt(idxStr, 10, 32) 125 126 fmt.Printf("Match file: %s index: %d n", name, idx) 127 128 data, _ := ioutil.ReadFile(tmpPath) 129 130 lastFile.WriteAt(data, idx*mbf.BlockSize) 131 132 os.Remove(tmpPath) 133 } 134 unlockFile(lastFile) 135 } 136 137 return nil 138 } 139 140 func indexHandle(w http.ResponseWriter, r *http.Request) { 141 tmp, _ := template.ParseFiles("./static/index.html") 142 tmp.Execute(w, "Index") 143 } 144 145 func uploadHandle(w http.ResponseWriter, r *http.Request) { 146 147 var mbf MultilBlockFile 148 mbf.FileName = r.FormValue("file_name") 149 mbf.Size, _ = strconv.ParseInt(r.FormValue("file_size"), 10, 64) 150 mbf.BlockSize, _ = strconv.ParseInt(r.FormValue("block_size"), 10, 64) 151 mbf.BreakError, _ = strconv.ParseBool(r.FormValue("break_error")) 152 153 var i int64 154 i, _ = strconv.ParseInt(r.FormValue("total_blocks"), 10, 32) 155 mbf.TotalBlocks = int(i) 156 157 i, _ = strconv.ParseInt(r.FormValue("index"), 10, 32) 158 mbf.Index = int(i) 159 160 d, _, _ := r.FormFile("data") 161 mbf.Bufs, _ = ioutil.ReadAll(d) 162 163 fmt.Printf(">>> Upload --------------------- n") 164 fmt.Printf("File name: %s n", mbf.FileName) 165 fmt.Printf("Size: %d n", mbf.Size) 166 fmt.Printf("Block size: %d n", mbf.BlockSize) 167 fmt.Printf("Total blocks: %d n", mbf.TotalBlocks) 168 fmt.Printf("Index: %d n", mbf.Index) 169 fmt.Println("Bufs len:", len(mbf.Bufs)) 170 171 multilBlockFile, _ := strconv.ParseBool(r.FormValue("multil_block")) 172 173 var err error 174 if multilBlockFile { 175 err = multilBlocksSave(&mbf) 176 } else { 177 err = singleFileSave(&mbf) 178 } 179 180 if !mbf.BreakError && err != nil { 181 w.WriteHeader(400) 182 fmt.Fprintf(w, fmt.Sprintf("%s", err)) 183 return 184 } 185 186 fmt.Fprintf(w, "ok") 187 } 188 189 func main() { 190 println("Listen on 8080") 191 192 http.HandleFunc("/", indexHandle) 193 http.HandleFunc("/upload", uploadHandle) 194 195 log.Fatal(http.ListenAndServe(":8080", nil)) 196 }
View Code
