commit 819361c3018594eea0f158e0414779f9f0812739
parent 6890cca621d442b16c78d92db77bf35a252da222
Author: n0tr1v <n0tr1v@protonmail.com>
Date: Mon, 30 Jan 2023 21:26:01 -0800
get file metadata
Diffstat:
2 files changed, 79 insertions(+), 27 deletions(-)
diff --git a/cmd/dkfupload/main.go b/cmd/dkfupload/main.go
@@ -3,11 +3,13 @@ package main
import (
"bytes"
"context"
+ "crypto/sha256"
"encoding/hex"
"flag"
"fmt"
"github.com/dustin/go-humanize"
"golang.org/x/net/proxy"
+ "io"
"log"
"math"
"math/rand"
@@ -15,7 +17,9 @@ import (
"net"
"net/http"
"net/http/cookiejar"
+ "net/url"
"os"
+ "strconv"
"strings"
"sync"
"time"
@@ -33,19 +37,27 @@ func main() {
var filedropUUID string
var fileName string
var dry bool
- var local bool
+ var isLocal bool
var maxChunkSize int64
- flag.StringVar(&filedropUUID, "filedrop-uuid", "", "dkf filedrop uuid")
- flag.StringVar(&filedropUUID, "u", "", "dkf filedrop uuid")
- flag.StringVar(&fileName, "file", "", "file to upload")
- flag.StringVar(&fileName, "f", "", "file to upload")
- flag.IntVar(&nbThreads, "threads", 20, "nb threads")
- flag.Int64Var(&maxChunkSize, "chunk-size", 10<<20, "chunk size") // 10MB
+ filedropUUIDUsage := "dkf filedrop uuid"
+ fileNameUsage := "file to upload"
+ nbThreadsUsage := "nb threads"
+ nbThreadsDefaultValue := 20
+ chunkSizeUsage := "chunk size"
+ chunkSizeDefaultValue := int64(10 << 20)
+ flag.StringVar(&filedropUUID, "filedrop-uuid", "", filedropUUIDUsage)
+ flag.StringVar(&filedropUUID, "u", "", filedropUUIDUsage)
+ flag.StringVar(&fileName, "file", "", fileNameUsage)
+ flag.StringVar(&fileName, "f", "", fileNameUsage)
+ flag.IntVar(&nbThreads, "threads", nbThreadsDefaultValue, nbThreadsUsage)
+ flag.IntVar(&nbThreads, "t", nbThreadsDefaultValue, nbThreadsUsage)
+ flag.Int64Var(&maxChunkSize, "chunk-size", chunkSizeDefaultValue, chunkSizeUsage) // 10MB
+ flag.Int64Var(&maxChunkSize, "c", chunkSizeDefaultValue, chunkSizeUsage) // 10MB
flag.BoolVar(&dry, "dry", false, "dry run")
- flag.BoolVar(&local, "local", false, "localhost development")
+ flag.BoolVar(&isLocal, "local", false, "localhost development")
flag.Parse()
- baseUrl := Ternary(local, localhostAddr, dkfBaseURL)
+ baseUrl := Ternary(isLocal, localhostAddr, dkfBaseURL)
f, err := os.Open(fileName)
if err != nil {
@@ -55,11 +67,19 @@ func main() {
if err != nil {
log.Fatalln(err.Error())
}
+
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ log.Fatalln(err)
+ }
+ fileSha256 := hex.EncodeToString(h.Sum(nil))
+
fileSize := fs.Size()
nbChunks := int64(math.Ceil(float64(fileSize) / float64(maxChunkSize)))
log.Printf("filedrop UUID: %s\n", filedropUUID)
log.Printf(" file: %s\n", fs.Name())
+ log.Printf(" sha256: %s\n", fileSha256)
log.Printf(" file size: %s (%s)\n", humanize.Bytes(uint64(fileSize)), humanize.Comma(fileSize))
log.Printf(" chunks size: %s (%s)\n", humanize.Bytes(uint64(maxChunkSize)), humanize.Comma(maxChunkSize))
log.Printf(" nb chunks: %d\n", nbChunks)
@@ -69,6 +89,25 @@ func main() {
}
log.Println(strings.Repeat("-", 80))
+ {
+ client := doGetClient(isLocal)
+ body := url.Values{}
+ body.Set("fileName", fs.Name())
+ body.Set("fileSize", strconv.FormatInt(fileSize, 10))
+ body.Set("fileSha256", fileSha256)
+ req, _ := http.NewRequest(http.MethodPost, baseUrl+"/file-drop/"+filedropUUID+"/tmp-init", strings.NewReader(body.Encode()))
+ req.Header.Set("User-Agent", userAgent)
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ resp, err := client.Do(req)
+ if err != nil {
+ log.Fatalln(err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ log.Fatalln(fmt.Errorf("invalid status code %s", resp.Status))
+ }
+ }
+
chunksCh := make(chan int64)
go func() {
for chunkNum := int64(0); chunkNum < nbChunks; chunkNum++ {
@@ -82,20 +121,7 @@ func main() {
for i := 0; i < nbThreads; i++ {
go func(i int, wg *sync.WaitGroup, chunksCh chan int64) {
- var client *http.Client
- hasToSucceed(func() error {
- username := fmt.Sprintf("user_%d", i)
- password := GenerateTokenN(8)
- if local {
- client = http.DefaultClient
- } else {
- client, err = GetHttpClient(&proxy.Auth{User: username, Password: password})
- if err != nil {
- return err
- }
- }
- return nil
- })
+ client := doGetClient(isLocal)
buf := make([]byte, maxChunkSize)
for chunkNum := range chunksCh {
@@ -142,6 +168,23 @@ func main() {
log.Printf("All done\n")
}
+func doGetClient(isLocal bool) (client *http.Client) {
+ hasToSucceed(func() error {
+ if isLocal {
+ client = http.DefaultClient
+ } else {
+ token := GenerateTokenN(8)
+ var err error
+ client, err = GetHttpClient(&proxy.Auth{User: token, Password: token})
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+ return
+}
+
// Will keep retrying a callback until no error is returned
func hasToSucceed(clb func() error) {
waitTime := 5
diff --git a/pkg/web/handlers/handlers.go b/pkg/web/handlers/handlers.go
@@ -4306,6 +4306,19 @@ func FileDropHandler(c echo.Context) error {
}
func FileDropTmpInitHandler(c echo.Context) error {
+ filedropUUID := c.Param("uuid")
+ if err := os.Mkdir(filepath.Join(config.Global.ProjectFiledropPath(), filedropUUID), 0755); err != nil {
+ logrus.Error(err)
+ return c.NoContent(http.StatusInternalServerError)
+ }
+ fileName := c.Request().PostFormValue("fileName")
+ fileSize := c.Request().PostFormValue("fileSize")
+ fileSha256 := c.Request().PostFormValue("fileSha256")
+ data := []byte(fileName + "\n" + fileSize + "\n" + fileSha256)
+ if err := os.WriteFile(filepath.Join(config.Global.ProjectFiledropPath(), filedropUUID, "metadata"), data, 0644); err != nil {
+ logrus.Error(err)
+ return c.NoContent(http.StatusInternalServerError)
+ }
return c.NoContent(http.StatusOK)
}
@@ -4319,10 +4332,6 @@ func FileDropTmpHandler(c echo.Context) error {
defer file.Close()
fileName := handler.Filename
by, _ := io.ReadAll(file)
- if err := os.MkdirAll(filepath.Join(config.Global.ProjectFiledropPath(), filedropUUID), 0755); err != nil {
- logrus.Error(err)
- return c.NoContent(http.StatusInternalServerError)
- }
p := filepath.Join(config.Global.ProjectFiledropPath(), filedropUUID, fileName)
if err := os.WriteFile(p, by, 0644); err != nil {
logrus.Error(err)
