FormData/Go fragment / block file upload

The FormData interface provides a way to construct key value pairs that represent form data, and the data passing through it can be used XMLHttpRequest.send() Method, this interface and this method are quite simple and direct. If the code type at the time of delivery is set to "multipart / form data", it will use the same format as the form.

If you want to build a simple GET request and use the <form> With query parameters, you can pass it directly to the URLSearchParams.

More explanation MDN: https://developer.mozilla.org/zh-CN/docs/Web/API/FormData

The file upload is divided into two parts: the file is divided into two parts.

1. The front-end js uses file.slice to cut piece by piece data from the file, then uses FormData to package it, and uses XMLHttpRequest to send the cut data block by block to the server

2. Each block received by the server is a multipart / form data form. You can put a lot of auxiliary information, file name, size, block size, block index in the form, and always bring the binary data cut out.

Multipart / form data data

POST /upload HTTP/1.1
Host: localhost:8080
Content-Length: 2098072
User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36

Content-Type: multipart/form-data; boundary=----WebKitFormBoundarymtng0xrR3ASR7wx7

------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="file_name"

apache-maven-3.6.3-bin.zip
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="file_size"

9602303
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="block_size"

2097152
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="total_blocks"

5
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="break_error"

true
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="index"

3
------WebKitFormBoundaryHdBeczaB5xBq6d55
Content-Disposition: form-data; name="data"; filename="blob"
Content-Type: application/octet-stream
(binary)

There are basically two ways to store files on the Server:

1. Directly create a file of corresponding size and write it in according to the offset position of each piece of data.

2. Each transmitted data block is saved as a separate data block file, and finally all file blocks are merged into files.

I just made a simple demonstration code here, which can't be used in the production environment basically.

Index.html, write js directly

  1 <!DOCTYPE html>
  2 <html>
  3     <head>
  4         <meta charset="utf8">
  5         <title>Multil-Blocks upload</title>
  6     </head>
  7 
  8     <body>
  9         <h2>Multil-Blocks upload</h2>
 10 
 11         <input id="file" type="file" />
 12 
 13         <input type="checkbox" id="multil_block_file">multil block file</input>
 14         <button type="button" onclick="on_block_upload()">Block upload</button>
 15         <button type="button" onclick="on_concurrency_upload()">Concurrency upload</button>
 16         <hr/>
 17 
 18         <div>
 19             <label>File name: </label><span id="file_name"></span>
 20         </div>
 21         <div>
 22             <label>File size: </label><span id="file_size"></span>
 23         </div>
 24         <div>
 25             <label>Split blocks: </label><span id="block_count"></span>
 26         </div>
 27 
 28         <hr/>
 29 
 30         <p id="upload_info"></p>
 31 
 32         <script>
 33             var Block_Size = 1024 * 1024 * 2;
 34 
 35             var el_file = document.getElementById('file');
 36             var el_multil_block_file = document.getElementById('multil_block_file');
 37             var el_file_name = document.getElementById('file_name');
 38             var el_file_size = document.getElementById('file_size');
 39             var el_block_count = document.getElementById('block_count');
 40             var el_upload_info = document.getElementById('upload_info');
 41 
 42             var file = null;
 43             var total_blocks = 0;
 44             var block_index = -1;
 45             var block_index_random_arr = [];
 46             var form_data = null;
 47 
 48 
 49             el_file.onchange = function() {
 50                 if (this.files.length === 0) return;
 51 
 52                 file = this.files[0];
 53                 total_blocks = Math.ceil( file.size / Block_Size );
 54 
 55                 el_file_name.innerText = file.name;
 56                 el_file_size.innerText = file.size;
 57                 el_block_count.innerText = total_blocks;
 58             }
 59 
 60             function print_info(msg) {
 61                 el_upload_info.innerHTML += `${msg}<br/>`;
 62             }
 63 
 64             function done() {
 65                 file = null;
 66                 total_blocks = 0;
 67                 block_index = -1;
 68                 form_data = null;
 69                 
 70                 el_file.value = '';
 71             }
 72 
 73 
 74             function get_base_form_data() {
 75                 var base_data = new FormData();
 76                 base_data.append('file_name', file.name);
 77                 base_data.append('file_size', file.size);
 78                 base_data.append('block_size', Block_Size);
 79                 base_data.append('total_blocks', total_blocks);
 80                 base_data.append('break_error', true);
 81                 base_data.append('index', 0);
 82                 base_data.append('data', null);
 83 
 84                 return base_data
 85             }
 86 
 87 
 88             function build_block_index_random_arr() {
 89                 block_index_random_arr = new Array(total_blocks).fill(0).map((v,i) => i);
 90                 block_index_random_arr.sort((n, m) => Math.random() > .5 ? -1 : 1);
 91 
 92                 print_info(`Upload sequence: ${block_index_random_arr}`);
 93             }
 94 
 95 
 96             function post(index, success_cb, failed_cb) {
 97                 if (!form_data) {
 98                     form_data = get_base_form_data();
 99                 }
100                 var start = index * Block_Size;
101                 var end = Math.min(file.size, start + Block_Size);
102 
103                 form_data.set('index', index);
104                 form_data.set('data', file.slice(start, end));
105 
106                 print_info(`Post ${index}/${total_blocks}, offset: ${start} -- ${end}`);
107 
108 
109                 var xhr = new XMLHttpRequest();
110                 xhr.open('POST', '/upload', true);
111                 /*
112                     Browser-based general content types
113                     Content-Type: multipart/form-data; boundary=----WebKitFormBoundarysXH5DIES2XFMuLXL
114 
115                     Error content type:
116                     xhr.setRequestHeader('Content-Type', 'multipart/form-data');
117                     Content-Type: multipart/form-data;
118                 */
119                 xhr.onreadystatechange = function() {
120 
121                     if (xhr.readyState === XMLHttpRequest.DONE) {
122 
123                         if (xhr.status >= 200 && xhr.status < 300 && success_cb) {
124                             return success_cb();
125                         }
126 
127                         if (xhr.status >= 400 && failed_cb) {
128                             failed_cb();
129                         }
130                     }
131                 }
132 
133                 // xhr.onerror event
134                 xhr.send(form_data);
135             }
136 
137 
138             function block_upload() {
139                 if (!file) {
140                     return;
141                 }
142                 if (block_index + 1 >= total_blocks) {
143                     return done();
144                 }
145 
146                 block_index++;
147                 var index = block_index_random_arr[block_index];
148             
149                 post(index, block_upload);
150             }
151 
152 
153             function concurrency_upload() {
154                 if (!file || total_blocks === 0) {
155                     return;
156                 }
157 
158                 build_block_index_random_arr();
159 
160                 form_data = get_base_form_data();
161                 form_data.set('break_error', false);
162                 form_data.set('multil_block', el_multil_block_file.checked);
163 
164                 for (var i of block_index_random_arr) {
165                     ((idx) => {
166                         post(idx, null, function() {
167                             print_info(`Failed: ${idx}`);
168                             setTimeout(() => post(idx), 1000);
169                         });
170                     })(i);
171                 }
172             }
173 
174 
175             function on_block_upload() {
176                 if (file) {
177                     print_info('Block upload');
178 
179                     form_data = get_base_form_data();
180                     form_data.set('multil_block', el_multil_block_file.checked);
181 
182                     build_block_index_random_arr();
183 
184                     block_index = -1;
185                     block_upload();
186                 }
187             }
188 
189             function on_concurrency_upload() {
190                 if (file) {
191                     print_info('Concurrency upload');
192                     concurrency_upload();
193                 }
194             }
195         </script>
196 
197     </body>
198 </html>

Simple Go server and save file, basically ignore all error handling

  1 package main
  2 
  3 import (
  4     "fmt"
  5     "io/ioutil"
  6     "log"
  7     "net/http"
  8     "os"
  9     "path"
 10     "path/filepath"
 11     "regexp"
 12     "strconv"
 13     "strings"
 14     "syscall"
 15     "text/template"
 16 )
 17 
 18 type MultilBlockFile struct {
 19     FileName    string
 20     Size        int64
 21     BlockSize   int64
 22     TotalBlocks int
 23     Index       int
 24     Bufs        []byte
 25     BreakError  bool
 26 }
 27 
 28 func fileIsExist(f string) bool {
 29     _, err := os.Stat(f)
 30     return err == nil || os.IsExist(err)
 31 }
 32 
 33 func lockFile(f *os.File) error {
 34     err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
 35     if err != nil {
 36         return fmt.Errorf("get flock failed. err: %s", err)
 37     }
 38 
 39     return nil
 40 }
 41 
 42 func unlockFile(f *os.File) error {
 43     defer f.Close()
 44     return syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
 45 }
 46 
 47 func singleFileSave(mbf *MultilBlockFile) error {
 48 
 49     dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
 50     filePath := path.Join(dir, "tmp", mbf.FileName)
 51 
 52     offset := int64(mbf.Index) * mbf.BlockSize
 53 
 54     fmt.Println(">>> Single file save ---------------------")
 55     fmt.Printf("Save file: %s \n", filePath)
 56     fmt.Printf("File offset: %d \n", offset)
 57 
 58     var f *os.File
 59     var needTruncate bool = false
 60     if !fileIsExist(filePath) {
 61         needTruncate = true
 62     }
 63 
 64     f, _ = os.OpenFile(filePath, syscall.O_CREAT|syscall.O_WRONLY, 0777)
 65 
 66     err := lockFile(f)
 67     if err != nil {
 68         if mbf.BreakError {
 69             log.Fatalf("get flock failed. err: %s", err)
 70         } else {
 71             return err
 72         }
 73     }
 74 
 75     if needTruncate {
 76         f.Truncate(mbf.Size)
 77     }
 78 
 79     f.WriteAt(mbf.Bufs, offset)
 80 
 81     unlockFile(f)
 82 
 83     return nil
 84 }
 85 
 86 func multilBlocksSave(mbf *MultilBlockFile) error {
 87     dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
 88     tmpFolderPath := path.Join(dir, "tmp")
 89     tmpFileName := fmt.Sprintf("%s.%d", mbf.FileName, mbf.Index)
 90     fileBlockPath := path.Join(tmpFolderPath, tmpFileName)
 91 
 92     f, _ := os.OpenFile(fileBlockPath, syscall.O_CREAT|syscall.O_WRONLY|syscall.O_TRUNC, 0777)
 93     defer f.Close()
 94 
 95     f.Write(mbf.Bufs)
 96     f.Close()
 97 
 98     re := regexp.MustCompile(`(?i:^` + mbf.FileName + `).\d$`)
 99 
100     files, _ := ioutil.ReadDir(tmpFolderPath)
101     matchFiles := make(map[string]bool)
102 
103     for _, file := range files {
104         if file.IsDir() {
105             continue
106         }
107 
108         fname := file.Name()
109         if re.MatchString(fname) {
110             matchFiles[fname] = true
111         }
112     }
113 
114     if len(matchFiles) >= mbf.TotalBlocks {
115         lastFile, _ := os.OpenFile(path.Join(tmpFolderPath, mbf.FileName), syscall.O_CREAT|syscall.O_WRONLY, 0777)
116         lockFile(lastFile)
117 
118         lastFile.Truncate(mbf.Size)
119 
120         for name := range matchFiles {
121             tmpPath := path.Join(tmpFolderPath, name)
122 
123             idxStr := name[strings.LastIndex(name, ".")+1:]
124             idx, _ := strconv.ParseInt(idxStr, 10, 32)
125 
126             fmt.Printf("Match file: %s index: %d \n", name, idx)
127 
128             data, _ := ioutil.ReadFile(tmpPath)
129 
130             lastFile.WriteAt(data, idx*mbf.BlockSize)
131 
132             os.Remove(tmpPath)
133         }
134         unlockFile(lastFile)
135     }
136 
137     return nil
138 }
139 
140 func indexHandle(w http.ResponseWriter, r *http.Request) {
141     tmp, _ := template.ParseFiles("./static/index.html")
142     tmp.Execute(w, "Index")
143 }
144 
145 func uploadHandle(w http.ResponseWriter, r *http.Request) {
146 
147     var mbf MultilBlockFile
148     mbf.FileName = r.FormValue("file_name")
149     mbf.Size, _ = strconv.ParseInt(r.FormValue("file_size"), 10, 64)
150     mbf.BlockSize, _ = strconv.ParseInt(r.FormValue("block_size"), 10, 64)
151     mbf.BreakError, _ = strconv.ParseBool(r.FormValue("break_error"))
152 
153     var i int64
154     i, _ = strconv.ParseInt(r.FormValue("total_blocks"), 10, 32)
155     mbf.TotalBlocks = int(i)
156 
157     i, _ = strconv.ParseInt(r.FormValue("index"), 10, 32)
158     mbf.Index = int(i)
159 
160     d, _, _ := r.FormFile("data")
161     mbf.Bufs, _ = ioutil.ReadAll(d)
162 
163     fmt.Printf(">>> Upload --------------------- \n")
164     fmt.Printf("File name: %s \n", mbf.FileName)
165     fmt.Printf("Size: %d \n", mbf.Size)
166     fmt.Printf("Block size: %d \n", mbf.BlockSize)
167     fmt.Printf("Total blocks: %d \n", mbf.TotalBlocks)
168     fmt.Printf("Index: %d \n", mbf.Index)
169     fmt.Println("Bufs len:", len(mbf.Bufs))
170 
171     multilBlockFile, _ := strconv.ParseBool(r.FormValue("multil_block"))
172 
173     var err error
174     if multilBlockFile {
175         err = multilBlocksSave(&mbf)
176     } else {
177         err = singleFileSave(&mbf)
178     }
179 
180     if !mbf.BreakError && err != nil {
181         w.WriteHeader(400)
182         fmt.Fprintf(w, fmt.Sprintf("%s", err))
183         return
184     }
185 
186     fmt.Fprintf(w, "ok")
187 }
188 
189 func main() {
190     println("Listen on 8080")
191 
192     http.HandleFunc("/", indexHandle)
193     http.HandleFunc("/upload", uploadHandle)
194 
195     log.Fatal(http.ListenAndServe(":8080", nil))
196 }

Screenshot of the directory, it's confusing

Tags: Go Mac OS X Apache Maven

Posted on Thu, 26 Mar 2020 10:00:10 -0400 by johncal