diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..40edc1c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +ipfsStorage +reposStorage diff --git a/README.md b/README.md new file mode 100644 index 0000000..56bb49b --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# padArchiver +Tool to store a pad (from the link) into IPFS and Git. + + + +## padArchiver-APIserver +This is an API to run in localhost. + +#### Run +To run using the compiled binary: +- The Git repo needs to be initialized, and with the remote already configured. +- The IPFS daemon needs to be running: +``` +> ipfs daemon +``` + +- Edit the file config.json to configure the desired port: +``` +{ + "port": "3080" +} +``` + +- Execute the API server: +``` +> ./padArchiver-APIserver +``` + +#### API Routes + +###### - GET /repos +this returns: +``` +[ + 'repo01', + 'repo02' +] +``` + + +###### - GET /repos/{repoid} +this returns: +``` +[ + 'repo01', + 'repo01/Group1', + 'repo01/Group1/Pad1.md', + 'repo01/Group2', + 'repo01/Group2/Pad2.md', + 'repo01/Group2/Pad3.md', + 'repo02/GroupA/Pad1.md' +] +``` + + +###### - POST /repos/{repoid}/pad +data to send: +``` +json: { + "link": "http://board.net/p/pad1", + "dir": "Group1", + "title": "Pad1" +} +``` +this returns: +``` +{ + "link": "http://board.net/p/pad1", + "dir": "Group1", + "title": "Pad1", + "ipfsHash": "QmVyp4JSREK5syLmNRCafkZkhzC7CfvS9qYWKfvfffqK2B" +} +``` +The IPFS hash is also added to the first line of the document, before adding the document to Git. + +## padArchiver-cli +To run the CLI, just need to run: +``` +./padArchiver-cli +``` +And follow the instructions. diff --git a/error.go b/error.go new file mode 100644 index 0000000..48b89c8 --- /dev/null +++ b/error.go @@ -0,0 +1,9 @@ +package padArchiver + +import "github.com/fatih/color" + +func check(err error) { + if err != nil { + color.Red(err.Error()) + } +} diff --git a/etherpad.go b/etherpad.go new file mode 100644 index 0000000..71a25d7 --- /dev/null +++ b/etherpad.go @@ -0,0 +1,46 @@ +package padArchiver + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + "os" +) + +func (repo *Repo) GetPad(link string, extension string, directory string, title string) (string, error) { + if extension != "md" && extension != "txt" && extension != "html" && extension != "pdf" && extension != "odt" { + return "", errors.New("No valid extension") + } + format := extension + if extension == "md" { + format = "markdown" + extension = "md" + } + + //create the pads directory + _ = os.Mkdir(repo.Dir+"/"+directory, os.ModePerm) + + completeLink := link + "/export/" + format + + //get the content from the url + r, err := http.Get(completeLink) + if err != nil { + fmt.Println(err) + return "", err + } + defer r.Body.Close() + content, err := ioutil.ReadAll(r.Body) + if err != nil { + fmt.Printf("%s", err) + return "", err + } + + //save the content into a file + err = ioutil.WriteFile(repo.Dir+"/"+directory+"/"+title+"."+extension, content, 0644) + if err != nil { + fmt.Println(err) + return "", err + } + return repo.Dir + "/" + directory + "/" + title + "." + extension, nil +} diff --git a/file.go b/file.go new file mode 100644 index 0000000..afdc668 --- /dev/null +++ b/file.go @@ -0,0 +1,21 @@ +package padArchiver + +import ( + "io/ioutil" +) + +func AddLineToFile(path string, line string) error { + fileBytes, err := ioutil.ReadFile(path) + if err != nil { + return err + } + content := string(fileBytes) + r := line + "\n\n" + r = r + content + + err = ioutil.WriteFile(path, []byte(r), 0644) + if err != nil { + return err + } + return nil +} diff --git a/git.go b/git.go new file mode 100644 index 0000000..42836c4 --- /dev/null +++ b/git.go @@ -0,0 +1,26 @@ +package padArchiver + +import ( + "fmt" + "os/exec" +) + +//TODO this is not finished +func (repo *Repo) GitUpdate(commitMsg string) error { + _, err := exec.Command("bash", "-c", "git pull origin master").Output() + if err != nil { + fmt.Println(err) + return err + } + _, err = exec.Command("bash", "-c", "git add .").Output() + if err != nil { + fmt.Println(err) + return err + } + _, err = exec.Command("bash", "-c", "git commit -m '"+commitMsg+"'").Output() + if err != nil { + fmt.Println(err) + return err + } + return nil +} diff --git a/ipfs.go b/ipfs.go new file mode 100644 index 0000000..9fef6c1 --- /dev/null +++ b/ipfs.go @@ -0,0 +1,41 @@ +package padArchiver + +import ( + "fmt" + "os" + + sh "github.com/ipfs/go-ipfs-api" +) + +//GettedPads is the directory where are stored the pads that are getted from IPFS +const IpfsStorage = "ipfsStorage" +const GettedPads = "ipfsStorage/gettedPads" + +//Add gets the content from the etherpad specified in the link, and downloads it in the format of the specified extension, and then, puts it into IPFS +func IpfsAdd(path string) (string, error) { + //connect to ipfs shell + s := sh.NewShell("localhost:5001") + //save the file into IPFS + ipfsHash, err := s.AddDir(path) + if err != nil { + fmt.Println(err) + return "", err + } + return ipfsHash, nil +} + +//Get gets the content from IPFS for a given hash, and saves it into a file +func IpfsGet(hash string, filename string) error { //create the pads directory + //create the pads directory + _ = os.Mkdir(IpfsStorage, os.ModePerm) + _ = os.Mkdir(GettedPads, os.ModePerm) + + //connect to ipfs shell + s := sh.NewShell("localhost:5001") + err := s.Get(hash, GettedPads+"/"+filename) + if err != nil { + fmt.Println(err) + return err + } + return nil +} diff --git a/padArchiver-APIserver/config.json b/padArchiver-APIserver/config.json new file mode 100644 index 0000000..45c2ff9 --- /dev/null +++ b/padArchiver-APIserver/config.json @@ -0,0 +1,3 @@ +{ + "port": "3080" +} diff --git a/padArchiver-APIserver/error.go b/padArchiver-APIserver/error.go new file mode 100644 index 0000000..e997ec1 --- /dev/null +++ b/padArchiver-APIserver/error.go @@ -0,0 +1,9 @@ +package main + +import "log" + +func check(err error) { + if err != nil { + log.Println(err) + } +} diff --git a/padArchiver-APIserver/main.go b/padArchiver-APIserver/main.go new file mode 100644 index 0000000..37ae167 --- /dev/null +++ b/padArchiver-APIserver/main.go @@ -0,0 +1,100 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "os" + "path/filepath" + "strings" + + padArchiver ".." + "github.com/gorilla/mux" +) + +type PadModel struct { + Link string `json:"link"` + Dir string `json:"dir"` + Title string `json:"title"` + IpfsHash string `json:"ipfsHash"` +} +type Repo struct { + Pads []string `json:"pads"` +} + +func main() { + readConfig("config.json") + + router := mux.NewRouter() + router.HandleFunc("/repos", GetReposList).Methods("GET") + router.HandleFunc("/repos/{repoid}", GetRepoIDList).Methods("GET") + router.HandleFunc("/repos/{repoid}/pad", PostStorePad).Methods("POST") + + log.Println("padArchiver API server running") + log.Print("port: ") + log.Println(config.Port) + log.Fatal(http.ListenAndServe(":"+config.Port, router)) +} + +func GetReposList(w http.ResponseWriter, r *http.Request) { + file, err := os.Open(padArchiver.Storage) + if err != nil { + log.Fatalf("failed opening directory: %s", err) + } + defer file.Close() + + list, _ := file.Readdirnames(0) // 0 to read all files and folders + + var repos []string + for _, name := range list { + repos = append(repos, strings.Replace(name, padArchiver.Storage+"/", "", -1)) + } + + jResp, err := json.Marshal(repos) + check(err) + fmt.Fprintln(w, string(jResp)) +} + +func GetRepoIDList(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + repoid := vars["repoid"] + + fileList := []string{} + err := filepath.Walk(padArchiver.Storage+"/"+repoid, func(path string, f os.FileInfo, err error) error { + fileList = append(fileList, path) + return nil + }) + + var files []string + for _, file := range fileList { + files = append(files, strings.Replace(file, padArchiver.Storage+"/", "", -1)) + } + jResp, err := json.Marshal(files) + check(err) + fmt.Fprintln(w, string(jResp)) +} +func PostStorePad(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + repoid := vars["repoid"] + + //open the repo + repo := padArchiver.OpenRepo(repoid) + //get the pad json + var pad PadModel + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(&pad) + if err != nil { + panic(err) + } + defer r.Body.Close() + + ipfsHash, err := repo.StorePad(pad.Link, pad.Dir, pad.Title) + if err != nil { + http.Error(w, "error storing pad", http.StatusConflict) + } + pad.IpfsHash = ipfsHash + jResp, err := json.Marshal(pad) + check(err) + fmt.Fprintln(w, string(jResp)) +} diff --git a/padArchiver-APIserver/padArchiver-APIserver b/padArchiver-APIserver/padArchiver-APIserver new file mode 100755 index 0000000..19c7b2e Binary files /dev/null and b/padArchiver-APIserver/padArchiver-APIserver differ diff --git a/padArchiver-APIserver/readConfig.go b/padArchiver-APIserver/readConfig.go new file mode 100644 index 0000000..14fd1ad --- /dev/null +++ b/padArchiver-APIserver/readConfig.go @@ -0,0 +1,19 @@ +package main + +import ( + "encoding/json" + "io/ioutil" +) + +type Config struct { + Port string `json:"port"` +} + +var config Config + +func readConfig(path string) { + file, err := ioutil.ReadFile(path) + check(err) + content := string(file) + json.Unmarshal([]byte(content), &config) +} diff --git a/padArchiver-APIserver/test.py b/padArchiver-APIserver/test.py new file mode 100644 index 0000000..9211ff9 --- /dev/null +++ b/padArchiver-APIserver/test.py @@ -0,0 +1,48 @@ +''' + this script uses requests and provoj + provoj is a simple library to test endpoints of an API RESTful + provoj can be downloaded using: pip install provoj + provoj repository: https://github.com/arnaucode/provoj + + To run this test, just need to run: + python test.py +''' +import provoj +import requests + + +test = provoj.NewTest("testing padArchiver API Server") + +url = "http://127.0.0.1:3080" + + + +jsonData = {"link": "http://board.net/p/pad1", "dir": "Group1", "title": "Pad1"} +r = requests.post(url + "/repos/repo01/pad", json=jsonData) +test.rStatus("POST add new pad", r) +print(r.json()) + +jsonData = {"link": "http://board.net/p/pad2", "dir": "Group2", "title": "Pad2"} +r = requests.post(url + "/repos/repo01/pad", json=jsonData) +test.rStatus("POST add new pad", r) +print(r.json()) + +jsonData = {"link": "http://board.net/p/pad3", "dir": "Group2", "title": "Pad3"} +r = requests.post(url + "/repos/repo01/pad", json=jsonData) +test.rStatus("POST add new pad", r) +print(r.json()) + + +r = requests.get(url + "/repos") +test.rStatus("GET repos list", r) +print(r.json()) +reposList = r.json() +testRepo = reposList[0] + +r = requests.get(url + "/repos/" + testRepo) +test.rStatus("GET repo " + testRepo + " list", r) +print(r.json()) + + + +test.printScores() diff --git a/padArchiver-cli/main.go b/padArchiver-cli/main.go new file mode 100644 index 0000000..2901c16 --- /dev/null +++ b/padArchiver-cli/main.go @@ -0,0 +1,112 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "strings" + + padArchiver ".." + "github.com/fatih/color" +) + +const checkIcon = "\xE2\x9C\x94 " + +func main() { + asciiart := ` +. +. _ _ _ +. | | /\ | | (_) + _ __ __ _ __| | / \ _ __ ___| |__ ___ _____ _ __ +| '_ \ / _ |/ _ | / /\ \ | '__/ __| '_ \| \ \ / / _ \ '__| +| |_) | (_| | (_| |/ ____ \| | | (__| | | | |\ V / __/ | +| .__/ \__,_|\__,_/_/ \_\_| \___|_| |_|_| \_/ \___|_| - cli +| | +|_| + + ` + color.Blue(asciiart) + fmt.Println(" v0.0.1") + color.Blue("https://github.com/arnaucode/padArchiver") + fmt.Println("") + fmt.Println("") + fmt.Println("") + + newcommand := bufio.NewReader(os.Stdin) + fmt.Print("Please select command number") + options := ` + 1 - Store Pad (to IPFS, Git, and send Telegram notification) + 2 - IPFS hash to file + 0 - Exit cli +option to select: ` + for { + fmt.Print(options) + + option, _ := newcommand.ReadString('\n') + option = strings.TrimSpace(option) + + switch option { + case "1": + fmt.Println("selected 1 - Store Pad (to IPFS and Git)") + option1() + break + case "2": + fmt.Println("selected 2 - IPFS hash to file") + option2() + break + case "0": + fmt.Println("selected 0 - exit cli") + os.Exit(3) + break + default: + fmt.Println("Invalid option") + break + } + } +} +func option1() { + newcommand := bufio.NewReader(os.Stdin) + fmt.Print(" Enter the repo ID (name): ") + repoID, _ := newcommand.ReadString('\n') + repoID = strings.Replace(repoID, "\n", "", -1) + + newcommand = bufio.NewReader(os.Stdin) + fmt.Print(" Enter the pad link: ") + link, _ := newcommand.ReadString('\n') + link = strings.Replace(link, "\n", "", -1) + + newcommand = bufio.NewReader(os.Stdin) + fmt.Print(" Enter the subdirectory: ") + subdirectory, _ := newcommand.ReadString('\n') + subdirectory = strings.Replace(subdirectory, "\n", "", -1) + + newcommand = bufio.NewReader(os.Stdin) + fmt.Print(" Enter the pad Title: ") + title, _ := newcommand.ReadString('\n') + title = strings.Replace(title, "\n", "", -1) + + repo := padArchiver.OpenRepo(repoID) + + ipfsHash, err := repo.StorePad(link, subdirectory, title) + if err != nil { + color.Red(err.Error()) + } else { + fmt.Println("IPFS hash: " + ipfsHash) + color.Green(checkIcon + "Pad stored in IPFS and Git") + } +} +func option2() { + newcommand := bufio.NewReader(os.Stdin) + fmt.Print(" Enter the IPFS hash: ") + hash, _ := newcommand.ReadString('\n') + hash = strings.Replace(hash, "\n", "", -1) + err := padArchiver.IpfsGet(hash, hash+".md") + if err != nil { + color.Red(err.Error()) + } else { + + color.Green(checkIcon + "File downloaded from IPFS network") + fmt.Print("File stored in: ") + color.Blue(padArchiver.GettedPads + "/" + hash + ".md") + } +} diff --git a/padArchiver-cli/padArchiver-cli b/padArchiver-cli/padArchiver-cli new file mode 100755 index 0000000..0b6cfba Binary files /dev/null and b/padArchiver-cli/padArchiver-cli differ diff --git a/padArchiver.go b/padArchiver.go new file mode 100644 index 0000000..f8bbec3 --- /dev/null +++ b/padArchiver.go @@ -0,0 +1,52 @@ +package padArchiver + +import ( + "os" + + "github.com/fatih/color" +) + +const Storage = "reposStorage" + +type Repo struct { + Dir string +} + +func OpenRepo(directory string) Repo { + //if not exist create the repos directory + _ = os.Mkdir(Storage, os.ModePerm) + + var repo Repo + repo.Dir = Storage + "/" + directory + //create the repo directory + _ = os.Mkdir(repo.Dir, os.ModePerm) + return repo +} + +func (repo *Repo) StorePad(link string, directory string, title string) (string, error) { + path, err := repo.GetPad(link, "md", directory, title) + if err != nil { + color.Red(err.Error()) + return "", err + } + + hash, err := IpfsAdd(path) + if err != nil { + color.Red(err.Error()) + return hash, err + } + + err = AddLineToFile(path, "IPFS hash of this document: "+hash) + if err != nil { + color.Red(err.Error()) + return hash, err + } + // TODO + // err = repo.GitUpdate("update commit") + // if err != nil { + // color.Red(err.Error()) + // return hash, err + // } + + return hash, nil +} diff --git a/padArchiver_test.go b/padArchiver_test.go new file mode 100644 index 0000000..eac4225 --- /dev/null +++ b/padArchiver_test.go @@ -0,0 +1,22 @@ +package padArchiver + +import ( + "testing" + + "github.com/fatih/color" +) + +const checkIcon = "\xE2\x9C\x94 " + +func TestAddPad(t *testing.T) { + color.Blue("TestAddPad") + repo := OpenRepo("Repo01") + _, err := repo.StorePad("http://board.net/p/pad1", "Group1", "pad1") + + if err == nil { + color.Green(checkIcon + "checked AddPad") + } else { + color.Red(err.Error()) + t.Errorf("Error AddPad") + } +}