feat(backend): add data export

backend/exporter/exporter.go

@@ -0,0 +1,265 @@
+package exporter
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"os/signal"
+	"sync"
+
+	"codeberg.org/u1f320/pronouns.cc/backend/db"
+	"codeberg.org/u1f320/pronouns.cc/backend/log"
+	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
+	"github.com/rs/xid"
+	"github.com/urfave/cli/v2"
+)
+
+var Command = &cli.Command{
+	Name:   "exporter",
+	Usage:  "Data exporter service",
+	Action: run,
+}
+
+type server struct {
+	Router chi.Router
+	DB     *db.DB
+
+	exporting   map[xid.ID]struct{}
+	exportingMu sync.Mutex
+}
+
+func run(c *cli.Context) error {
+	port := ":" + os.Getenv("EXPORTER_PORT")
+
+	db, err := db.New()
+	if err != nil {
+		log.Fatalf("creating database: %v", err)
+		return err
+	}
+
+	s := &server{
+		Router:    chi.NewRouter(),
+		DB:        db,
+		exporting: make(map[xid.ID]struct{}),
+	}
+
+	// set up middleware + the single route
+	s.Router.Use(middleware.Recoverer)
+	s.Router.Get("/start/{id}", s.startExport)
+
+	e := make(chan error)
+
+	// run server in another goroutine (for gracefully shutting down, see below)
+	go func() {
+		e <- http.ListenAndServe(port, s.Router)
+	}()
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer stop()
+
+	log.Infof("API server running at %v!", port)
+
+	select {
+	case <-ctx.Done():
+		log.Info("Interrupt signal received, shutting down...")
+		s.DB.Close()
+		return nil
+	case err := <-e:
+		log.Fatalf("Error running server: %v", err)
+	}
+
+	return nil
+}
+
+func (s *server) startExport(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	id, err := xid.FromString(chi.URLParam(r, "id"))
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	u, err := s.DB.User(ctx, id)
+	if err != nil {
+		log.Errorf("getting user %v: %v", id, err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	go s.doExport(u)
+
+	w.WriteHeader(http.StatusAccepted)
+}
+
+func (s *server) doExport(u db.User) {
+	s.exportingMu.Lock()
+	if _, ok := s.exporting[u.ID]; ok {
+		s.exportingMu.Unlock()
+		log.Debugf("user %v is already being exported, aborting", u.ID)
+		return
+	}
+	s.exporting[u.ID] = struct{}{}
+	s.exportingMu.Unlock()
+
+	defer func() {
+		s.exportingMu.Lock()
+		delete(s.exporting, u.ID)
+		s.exportingMu.Unlock()
+	}()
+
+	ctx := context.Background()
+
+	log.Debugf("[%v] starting export of user", u.ID)
+
+	outBuffer := new(bytes.Buffer)
+	zw := zip.NewWriter(outBuffer)
+	defer zw.Close()
+
+	w, err := zw.Create("user.json")
+	if err != nil {
+		log.Errorf("[%v] creating file in zip archive: %v", u.ID, err)
+		return
+	}
+
+	log.Debugf("[%v] getting user fields", u.ID)
+
+	fields, err := s.DB.UserFields(ctx, u.ID)
+	if err != nil {
+		log.Errorf("[%v] getting user fields: %v", u.ID, err)
+		return
+	}
+
+	log.Debugf("[%v] writing user json", u.ID)
+
+	ub, err := json.Marshal(dbUserToExport(u, fields))
+	if err != nil {
+		log.Errorf("[%v] marshaling user: %v", u.ID, err)
+		return
+	}
+
+	_, err = w.Write(ub)
+	if err != nil {
+		log.Errorf("[%v] writing user: %v", u.ID, err)
+		return
+	}
+
+	if u.Avatar != nil {
+		log.Debugf("[%v] getting user avatar", u.ID)
+
+		w, err := zw.Create("user_avatar.webp")
+		if err != nil {
+			log.Errorf("[%v] creating file in zip archive: %v", u.ID, err)
+			return
+		}
+
+		r, err := s.DB.UserAvatar(ctx, u.ID, *u.Avatar)
+		if err != nil {
+			log.Errorf("[%v] getting user avatar: %v", u.ID, err)
+			return
+		}
+		defer r.Close()
+
+		_, err = io.Copy(w, r)
+		if err != nil {
+			log.Errorf("[%v] writing user avatar: %v", u.ID, err)
+			return
+		}
+
+		log.Debugf("[%v] exported user avatar", u.ID)
+	}
+
+	members, err := s.DB.UserMembers(ctx, u.ID)
+	if err != nil {
+		log.Errorf("[%v] getting user members: %v", u.ID, err)
+		return
+	}
+
+	for _, m := range members {
+		log.Debugf("[%v] starting export for member %v", u.ID, m.ID)
+
+		fields, err := s.DB.MemberFields(ctx, m.ID)
+		if err != nil {
+			log.Errorf("[%v] getting fields for member %v: %v", u.ID, m.ID, err)
+			return
+		}
+
+		w, err := zw.Create("members/" + m.Name + "-" + m.ID.String() + ".json")
+		if err != nil {
+			log.Errorf("[%v] creating file in zip archive: %v", u.ID, err)
+			return
+		}
+
+		mb, err := json.Marshal(dbMemberToExport(m, fields))
+		if err != nil {
+			log.Errorf("[%v] marshaling member %v: %v", u.ID, m.ID, err)
+			return
+		}
+
+		_, err = w.Write(mb)
+		if err != nil {
+			log.Errorf("[%v] writing member %v json: %v", u.ID, m.ID, err)
+			return
+		}
+
+		if m.Avatar != nil {
+			log.Debugf("[%v] getting member %v avatar", u.ID, m.ID)
+
+			w, err := zw.Create("members/" + m.Name + "-" + m.ID.String() + "-avatar.webp")
+			if err != nil {
+				log.Errorf("[%v] creating file in zip archive: %v", u.ID, err)
+				return
+			}
+
+			r, err := s.DB.MemberAvatar(ctx, m.ID, *m.Avatar)
+			if err != nil {
+				log.Errorf("[%v] getting member %v avatar: %v", u.ID, m.ID, err)
+				return
+			}
+			defer r.Close()
+
+			_, err = io.Copy(w, r)
+			if err != nil {
+				log.Errorf("[%v] writing member %v avatar: %v", u.ID, m.ID, err)
+				return
+			}
+
+			log.Debugf("[%v] exported member %v avatar", u.ID, m.ID)
+		}
+
+		log.Debugf("[%v] finished export for member %v", u.ID, m.ID)
+	}
+
+	log.Debugf("[%v] finished export, uploading to object storage and saving in database", u.ID)
+
+	err = zw.Close()
+	if err != nil {
+		log.Errorf("[%v] closing zip file: %v", u.ID, err)
+		return
+	}
+
+	de, err := s.DB.CreateExport(ctx, u.ID, randomFilename(), outBuffer)
+	if err != nil {
+		log.Errorf("[%v] writing export: %v", u.ID, err)
+		return
+	}
+
+	log.Debugf("[%v] finished writing export. path: %q", u.ID, de.Path())
+}
+
+func randomFilename() string {
+	b := make([]byte, 32)
+
+	_, err := rand.Read(b)
+	if err != nil {
+		panic(err)
+	}
+
+	return base64.RawURLEncoding.EncodeToString(b)
+}