likes
comments
collection
share

大文件分段上传，前后端写法都有（分别使用node和golang实现）

作者站长头像
站长
· 阅读数 33

问题：为什么需要分段上传？

前端项目开发阶段是node环境，上传文件非常迅速，很大的文件也能轻松上传，但是生产环境一般都是用nginx部署，nginx对上传文件大小默认是1M，即使是修改文件大小限制后，由于传输速度问题，要是上传个几兆几十兆的文件问题不大，要是上百兆甚至几百兆，大多数情况会出现超时，导致上传失败，体验很差，分段上传就可以很大程度解决这个问题

大致思路

分段上传需要前后端配合，

前端部分：
  1. 首先是前端要把需要上传的大文件分成若干段，按序号命名，循环上传每一段
  2. 前端循环上传完所有分段后调用合并接口
后端部分：
  1. 递归循环接收前端上传的分段，新建一个和文件同名的文件夹，把接收到的分段都存入该文件夹
  2. 合并接口接收文件夹名为参数，把该文件夹下所有文件按序号排序后合并为一个文件
  3. 合并文件后在同目录生成合并后的文件，并删除分段文件夹

node版本

代码仓库 gitee.com/liyefeng123… 主要是两个接口，上传文件切片接口和合并切片接口 index.js 入口

const express=require("express")
const bodyparser=require("body-parser")
const multiparty=require("multiparty")
const fse=require("fs-extra")
const fs=require("fs")
const path = require('path')
const app =express()
const UPLOAD_DIR=path.resolve(__dirname,"public/upload")
app.use(express.static(__dirname+"/public"))
app.use(bodyparser.urlencoded({extended:true}))

app.post("/upload",function(req,res){
  const form=new multiparty.Form({uploadDir:"template"})
  form.parse(req)
  form.on("file",async(name,chunk)=>{
    
    let chunkDir=`${UPLOAD_DIR}/${chunk.originalFilename.split(".")[0]}`
    if(!fse.existsSync(chunkDir)){
      await fse.mkdirs(chunkDir)
    }
    var dpath=path.join(chunkDir,chunk.originalFilename.split(".")[1])
    await fse.move(chunk.path,dpath,{overwrite:true})
    res.send("文件上传成功")
  })
  
})
app.get("/merge", async function (req,res){
  let name=req.query.name
  let fname=name.split(".")[0]
  let chunDir=path.join(UPLOAD_DIR,fname)
  let chunks=await fse.readdir(chunDir)
  let sortArr=chunks.sort((a,b)=>a-b)
  sortArr.map(chunkPath=>{
    fs.appendFileSync(
      path.join(UPLOAD_DIR,name),
      fs.readFileSync(`${chunDir}/${chunkPath}`)
    )
  })
  fse.removeSync(chunDir)
  res.send({msg:"合并成功",url:`http://localhost:3000/upload/${name}`})
})
app.listen("3000")
console.log("listen 3000")

index.html

<!DOCTYPE html>
<html lang="en">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>文件上传</title>

</head>

<body>
  <input type="file" id="btnfile">
  <input type="button" value="上传" onclick="upload(0)">
  <button id="btn">0%</button>
  <script src="./axios.js"></script>
  <script>
    let chunkSize = 1024 * 1024
    let index=0
    let size=0
    let radio=0
    function upload(index) {
      let file = btnfile.files[0]
      console.log("file",file)
      size=file.size
      let [fname, fext] = file.name.split(".")
      
      let start = index * chunkSize
      radio=(index/(size/chunkSize))*100
      radio=Math.floor(radio)
      console.log("size/chunkSize",index/(size/chunkSize))
      btn.innerText=`${radio>100?100:radio}%`
      if (start > file.size) {
        merge(file.name)
        return
      }
      let blob = file.slice(start, start + chunkSize)
      let blobName = `${fname}.${index}.${fext}`
      let blobFile = new File([blob], blobName)
      let formData = new FormData()
      formData.append("file", blobFile)
      index++
      axios.post("/upload", formData).then(res => {
        console.log("rea", res)

        upload(index)
      })
    }
    function merge(name){
      axios.get("/merge?name="+name).then(res=>{
        console.log("merge",res)
      })
    }
    window.onload = () => {
      let btnfile = document.getElementById("btnfile")

    }
  </script>
</body>

</html>

golang版本

前端表单

<div class="upload_wrap" v-show="percentage<1">
  <input type="file" name="file" id="uploadVideo">
  <input type="button" value="开始上传" @click="uploadBiger">
</div>
<div class="loading"  style="width: 100%" v-show="percentage>1">
  <el-progress  :stroke-width="11" :percentage="percentage" />
</div>

大文件分段上传，前后端写法都有（分别使用node和golang实现）

/**
 * 上传大文件
 */
let uploadStep=ref(0)
let chunkSize = 1024 * 1024
let uploadSize=0
let index=0
let size=0
let percentage=ref(0)
const uploadBiger=()=>{
  const uploadVideo=document.getElementById("uploadVideo")
  let file=uploadVideo.files[0]
  size=file.size
  uploadSize=file.size
  let start = index * chunkSize
  let [fname, fext] = file.name.split(".")
  percentage.value=Math.floor((index/(size/chunkSize))*100)
  percentage.value=percentage.value>=100?100:percentage.value
  if (start > file.size) {
    mergeVideo(file.name)
    return
  }
  let blob = file.slice(start, start + chunkSize)
  let blobName = `${fname}.${index}.${fext}`
  let blobFile = new File([blob], blobName)
  let formData = new FormData()
  formData.append("file", blobFile)
  index++
  server.post("/admin/uploadVideo",formData).then(res=>{
    console.log("up",res)
    if(res.status==200){
      uploadBiger(index)
    }
  })
  
}
const mergeVideo=(fileName)=>{
  let data={
    fileName
  }
  server.post("/admin/mergeVideo",data).then(res=>{
    if(res.status==200){
      form.url=res.data.url
    }
  })
}

golang后端代码

func (this VideoController) UploadVideo(ctx *gin.Context) {
    file, err := ctx.FormFile("file")
    name := ctx.PostForm("name")
    println("name", name)
    if err == nil {
        dsn := path.Join("./static/upload/video/"+name+"/", file.Filename)
        if err := ctx.SaveUploadedFile(file, dsn); err == nil {
            ctx.JSON(200, gin.H{
                "success": true,
                "message": "上传成功",
            })
        } else {
            ctx.JSON(400, gin.H{
                "success": false,
                "message": "上传失败",
            })
        }
    } else {
        ctx.JSON(400, gin.H{
            "success": false,
            "message": "上传失败",
        })
    }
}
func (this VideoController) MergeVideos(ctx *gin.Context) {
    name := ctx.PostForm("fileName")
    fileName := strings.Split(name, ".")
    config, _ := ini.Load("./conf/app.ini")
    videolocal := config.Section("").Key("mlocal").String()
    dir := path.Join("./static/upload/video/" + fileName[0] + "/")
    files, _ := ioutil.ReadDir(dir)
    newFile, err := os.Create("./static/upload/video/" + name)
    defer newFile.Close()
    if err != nil {
        println("文件创建失败")
        return
    }
    sort.Slice(files, func(i, j int) bool {
        iN, _ := strconv.Atoi(strings.Split(files[i].Name(), ".")[0])
        jN, _ := strconv.Atoi(strings.Split(files[j].Name(), ".")[0])
        return iN < jN
    })
    multiWriter := io.MultiWriter(newFile)
    for _, fn := range files {
        f := dir + "/" + fn.Name()
        sliceFile, err := os.Open(f)
        if err != nil {
            println("合并文件失败")
            return
        }
        defer sliceFile.Close()
        _, err = io.Copy(multiWriter, sliceFile)
        if err != nil {
            println("复制文件1失败")
            return
        }
    }
    err2 := os.RemoveAll(dir)
    if err2 != nil {
        println("删除文件1失败")
        return
    }
    ctx.JSON(200, gin.H{
        "success": true,
        "mes":     "合并成功",
        "url":     videolocal + "/static/upload/video/" + name,
    })
}

方法写的有点简略，有感兴趣的小伙伴可以单聊

转载自:https://juejin.cn/post/7360510179304767497
评论
请登录