api rest, cli. IPFS part works. To do: Git part

.gitignore

@@ -0,0 +1,2 @@
+ipfsStorage
+reposStorage

README.md

@@ -0,0 +1,81 @@
+# padArchiver
+Tool to store a pad (from the link) into IPFS and Git.
+
+
+
+## padArchiver-APIserver
+This is an API to run in localhost.
+
+#### Run
+To run using the compiled binary:
+- The Git repo needs to be initialized, and with the remote already configured.
+- The IPFS daemon needs to be running:
+```
+> ipfs daemon
+```
+
+- Edit the file config.json to configure the desired port:
+```
+{
+  "port": "3080"
+}
+```
+
+- Execute the API server:
+```
+> ./padArchiver-APIserver
+```
+
+#### API Routes
+
+###### - GET /repos
+this returns:
+```
+[
+  'repo01',
+  'repo02'
+]
+```
+
+
+###### - GET /repos/{repoid}
+this returns:
+```
+[
+  'repo01',
+  'repo01/Group1',
+  'repo01/Group1/Pad1.md',
+  'repo01/Group2',
+  'repo01/Group2/Pad2.md',
+  'repo01/Group2/Pad3.md',
+  'repo02/GroupA/Pad1.md'
+]
+```
+
+
+###### - POST /repos/{repoid}/pad
+data to send:
+```
+json: {
+  "link": "http://board.net/p/pad1",
+  "dir": "Group1",
+  "title": "Pad1"
+}
+```
+this returns:
+```
+{
+  "link": "http://board.net/p/pad1",
+  "dir": "Group1",
+  "title": "Pad1",
+  "ipfsHash": "QmVyp4JSREK5syLmNRCafkZkhzC7CfvS9qYWKfvfffqK2B"
+}
+```
+The IPFS hash is also added to the first line of the document, before adding the document to Git.
+
+## padArchiver-cli
+To run the CLI, just need to run:
+```
+./padArchiver-cli
+```
+And follow the instructions.

error.go

@@ -0,0 +1,9 @@
+package padArchiver
+
+import "github.com/fatih/color"
+
+func check(err error) {
+	if err != nil {
+		color.Red(err.Error())
+	}
+}

etherpad.go

@@ -0,0 +1,46 @@
+package padArchiver
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+)
+
+func (repo *Repo) GetPad(link string, extension string, directory string, title string) (string, error) {
+	if extension != "md" && extension != "txt" && extension != "html" && extension != "pdf" && extension != "odt" {
+		return "", errors.New("No valid extension")
+	}
+	format := extension
+	if extension == "md" {
+		format = "markdown"
+		extension = "md"
+	}
+
+	//create the pads directory
+	_ = os.Mkdir(repo.Dir+"/"+directory, os.ModePerm)
+
+	completeLink := link + "/export/" + format
+
+	//get the content from the url
+	r, err := http.Get(completeLink)
+	if err != nil {
+		fmt.Println(err)
+		return "", err
+	}
+	defer r.Body.Close()
+	content, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		fmt.Printf("%s", err)
+		return "", err
+	}
+
+	//save the content into a file
+	err = ioutil.WriteFile(repo.Dir+"/"+directory+"/"+title+"."+extension, content, 0644)
+	if err != nil {
+		fmt.Println(err)
+		return "", err
+	}
+	return repo.Dir + "/" + directory + "/" + title + "." + extension, nil
+}

file.go

@@ -0,0 +1,21 @@
+package padArchiver
+
+import (
+	"io/ioutil"
+)
+
+func AddLineToFile(path string, line string) error {
+	fileBytes, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	content := string(fileBytes)
+	r := line + "\n\n"
+	r = r + content
+
+	err = ioutil.WriteFile(path, []byte(r), 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}

git.go

@@ -0,0 +1,26 @@
+package padArchiver
+
+import (
+	"fmt"
+	"os/exec"
+)
+
+//TODO this is not finished
+func (repo *Repo) GitUpdate(commitMsg string) error {
+	_, err := exec.Command("bash", "-c", "git pull origin master").Output()
+	if err != nil {
+		fmt.Println(err)
+		return err
+	}
+	_, err = exec.Command("bash", "-c", "git add .").Output()
+	if err != nil {
+		fmt.Println(err)
+		return err
+	}
+	_, err = exec.Command("bash", "-c", "git commit -m '"+commitMsg+"'").Output()
+	if err != nil {
+		fmt.Println(err)
+		return err
+	}
+	return nil
+}

ipfs.go

@@ -0,0 +1,41 @@
+package padArchiver
+
+import (
+	"fmt"
+	"os"
+
+	sh "github.com/ipfs/go-ipfs-api"
+)
+
+//GettedPads is the directory where are stored the pads that are getted from IPFS
+const IpfsStorage = "ipfsStorage"
+const GettedPads = "ipfsStorage/gettedPads"
+
+//Add gets the content from the etherpad specified in the link, and downloads it in the format of the specified extension, and then, puts it into IPFS
+func IpfsAdd(path string) (string, error) {
+	//connect to ipfs shell
+	s := sh.NewShell("localhost:5001")
+	//save the file into IPFS
+	ipfsHash, err := s.AddDir(path)
+	if err != nil {
+		fmt.Println(err)
+		return "", err
+	}
+	return ipfsHash, nil
+}
+
+//Get gets the content from IPFS for a given hash, and saves it into a file
+func IpfsGet(hash string, filename string) error { //create the pads directory
+	//create the pads directory
+	_ = os.Mkdir(IpfsStorage, os.ModePerm)
+	_ = os.Mkdir(GettedPads, os.ModePerm)
+
+	//connect to ipfs shell
+	s := sh.NewShell("localhost:5001")
+	err := s.Get(hash, GettedPads+"/"+filename)
+	if err != nil {
+		fmt.Println(err)
+		return err
+	}
+	return nil
+}

padArchiver-APIserver/config.json

@@ -0,0 +1,3 @@
+{
+  "port": "3080"
+}

padArchiver-APIserver/error.go

@@ -0,0 +1,9 @@
+package main
+
+import "log"
+
+func check(err error) {
+	if err != nil {
+		log.Println(err)
+	}
+}

padArchiver-APIserver/main.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	padArchiver ".."
+	"github.com/gorilla/mux"
+)
+
+type PadModel struct {
+	Link     string `json:"link"`
+	Dir      string `json:"dir"`
+	Title    string `json:"title"`
+	IpfsHash string `json:"ipfsHash"`
+}
+type Repo struct {
+	Pads []string `json:"pads"`
+}
+
+func main() {
+	readConfig("config.json")
+
+	router := mux.NewRouter()
+	router.HandleFunc("/repos", GetReposList).Methods("GET")
+	router.HandleFunc("/repos/{repoid}", GetRepoIDList).Methods("GET")
+	router.HandleFunc("/repos/{repoid}/pad", PostStorePad).Methods("POST")
+
+	log.Println("padArchiver API server running")
+	log.Print("port: ")
+	log.Println(config.Port)
+	log.Fatal(http.ListenAndServe(":"+config.Port, router))
+}
+
+func GetReposList(w http.ResponseWriter, r *http.Request) {
+	file, err := os.Open(padArchiver.Storage)
+	if err != nil {
+		log.Fatalf("failed opening directory: %s", err)
+	}
+	defer file.Close()
+
+	list, _ := file.Readdirnames(0) // 0 to read all files and folders
+
+	var repos []string
+	for _, name := range list {
+		repos = append(repos, strings.Replace(name, padArchiver.Storage+"/", "", -1))
+	}
+
+	jResp, err := json.Marshal(repos)
+	check(err)
+	fmt.Fprintln(w, string(jResp))
+}
+
+func GetRepoIDList(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	repoid := vars["repoid"]
+
+	fileList := []string{}
+	err := filepath.Walk(padArchiver.Storage+"/"+repoid, func(path string, f os.FileInfo, err error) error {
+		fileList = append(fileList, path)
+		return nil
+	})
+
+	var files []string
+	for _, file := range fileList {
+		files = append(files, strings.Replace(file, padArchiver.Storage+"/", "", -1))
+	}
+	jResp, err := json.Marshal(files)
+	check(err)
+	fmt.Fprintln(w, string(jResp))
+}
+func PostStorePad(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	repoid := vars["repoid"]
+
+	//open the repo
+	repo := padArchiver.OpenRepo(repoid)
+	//get the pad json
+	var pad PadModel
+	decoder := json.NewDecoder(r.Body)
+	err := decoder.Decode(&pad)
+	if err != nil {
+		panic(err)
+	}
+	defer r.Body.Close()
+
+	ipfsHash, err := repo.StorePad(pad.Link, pad.Dir, pad.Title)
+	if err != nil {
+		http.Error(w, "error storing pad", http.StatusConflict)
+	}
+	pad.IpfsHash = ipfsHash
+	jResp, err := json.Marshal(pad)
+	check(err)
+	fmt.Fprintln(w, string(jResp))
+}

padArchiver-APIserver/readConfig.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+)
+
+type Config struct {
+	Port string `json:"port"`
+}
+
+var config Config
+
+func readConfig(path string) {
+	file, err := ioutil.ReadFile(path)
+	check(err)
+	content := string(file)
+	json.Unmarshal([]byte(content), &config)
+}

padArchiver-APIserver/test.py

@@ -0,0 +1,48 @@
+'''
+    this script uses requests and provoj
+    provoj is a simple library to test endpoints of an API RESTful
+    provoj can be downloaded using: pip install provoj
+    provoj repository: https://github.com/arnaucode/provoj
+
+    To run this test, just need to run:
+    python test.py
+'''
+import provoj
+import requests
+
+
+test = provoj.NewTest("testing padArchiver API Server")
+
+url = "http://127.0.0.1:3080"
+
+
+
+jsonData = {"link": "http://board.net/p/pad1", "dir": "Group1", "title": "Pad1"}
+r = requests.post(url + "/repos/repo01/pad", json=jsonData)
+test.rStatus("POST add new pad", r)
+print(r.json())
+
+jsonData = {"link": "http://board.net/p/pad2", "dir": "Group2", "title": "Pad2"}
+r = requests.post(url + "/repos/repo01/pad", json=jsonData)
+test.rStatus("POST add new pad", r)
+print(r.json())
+
+jsonData = {"link": "http://board.net/p/pad3", "dir": "Group2", "title": "Pad3"}
+r = requests.post(url + "/repos/repo01/pad", json=jsonData)
+test.rStatus("POST add new pad", r)
+print(r.json())
+
+
+r = requests.get(url + "/repos")
+test.rStatus("GET repos list", r)
+print(r.json())
+reposList = r.json()
+testRepo = reposList[0]
+
+r = requests.get(url + "/repos/" + testRepo)
+test.rStatus("GET repo " + testRepo + " list", r)
+print(r.json())
+
+
+
+test.printScores()

padArchiver-cli/main.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	padArchiver ".."
+	"github.com/fatih/color"
+)
+
+const checkIcon = "\xE2\x9C\x94 "
+
+func main() {
+	asciiart := `
+.
+.                _                   _      _
+.               | |   /\            | |    (_)
+ _ __   __ _  __| |  /  \   _ __ ___| |__  ___   _____ _ __
+| '_ \ / _  |/ _  | / /\ \ | '__/ __| '_ \| \ \ / / _ \ '__|
+| |_) | (_| | (_| |/ ____ \| | | (__| | | | |\ V /  __/ |
+| .__/ \__,_|\__,_/_/    \_\_|  \___|_| |_|_| \_/ \___|_|		- cli
+| |
+|_|
+
+		`
+	color.Blue(asciiart)
+	fmt.Println("							v0.0.1")
+	color.Blue("https://github.com/arnaucode/padArchiver")
+	fmt.Println("")
+	fmt.Println("")
+	fmt.Println("")
+
+	newcommand := bufio.NewReader(os.Stdin)
+	fmt.Print("Please select command number")
+	options := `
+	1 - Store Pad (to IPFS, Git, and send Telegram notification)
+	2 - IPFS hash to file
+	0 - Exit cli
+option to select: `
+	for {
+		fmt.Print(options)
+
+		option, _ := newcommand.ReadString('\n')
+		option = strings.TrimSpace(option)
+
+		switch option {
+		case "1":
+			fmt.Println("selected 1 - Store Pad (to IPFS and Git)")
+			option1()
+			break
+		case "2":
+			fmt.Println("selected 2 - IPFS hash to file")
+			option2()
+			break
+		case "0":
+			fmt.Println("selected 0 - exit cli")
+			os.Exit(3)
+			break
+		default:
+			fmt.Println("Invalid option")
+			break
+		}
+	}
+}
+func option1() {
+	newcommand := bufio.NewReader(os.Stdin)
+	fmt.Print("	Enter the repo ID (name): ")
+	repoID, _ := newcommand.ReadString('\n')
+	repoID = strings.Replace(repoID, "\n", "", -1)
+
+	newcommand = bufio.NewReader(os.Stdin)
+	fmt.Print("	Enter the pad link: ")
+	link, _ := newcommand.ReadString('\n')
+	link = strings.Replace(link, "\n", "", -1)
+
+	newcommand = bufio.NewReader(os.Stdin)
+	fmt.Print("	Enter the subdirectory: ")
+	subdirectory, _ := newcommand.ReadString('\n')
+	subdirectory = strings.Replace(subdirectory, "\n", "", -1)
+
+	newcommand = bufio.NewReader(os.Stdin)
+	fmt.Print("	Enter the pad Title: ")
+	title, _ := newcommand.ReadString('\n')
+	title = strings.Replace(title, "\n", "", -1)
+
+	repo := padArchiver.OpenRepo(repoID)
+
+	ipfsHash, err := repo.StorePad(link, subdirectory, title)
+	if err != nil {
+		color.Red(err.Error())
+	} else {
+		fmt.Println("IPFS hash: " + ipfsHash)
+		color.Green(checkIcon + "Pad stored in IPFS and Git")
+	}
+}
+func option2() {
+	newcommand := bufio.NewReader(os.Stdin)
+	fmt.Print("	Enter the IPFS hash: ")
+	hash, _ := newcommand.ReadString('\n')
+	hash = strings.Replace(hash, "\n", "", -1)
+	err := padArchiver.IpfsGet(hash, hash+".md")
+	if err != nil {
+		color.Red(err.Error())
+	} else {
+
+		color.Green(checkIcon + "File downloaded from IPFS network")
+		fmt.Print("File stored in: ")
+		color.Blue(padArchiver.GettedPads + "/" + hash + ".md")
+	}
+}

padArchiver.go

@@ -0,0 +1,52 @@
+package padArchiver
+
+import (
+	"os"
+
+	"github.com/fatih/color"
+)
+
+const Storage = "reposStorage"
+
+type Repo struct {
+	Dir string
+}
+
+func OpenRepo(directory string) Repo {
+	//if not exist create the repos directory
+	_ = os.Mkdir(Storage, os.ModePerm)
+
+	var repo Repo
+	repo.Dir = Storage + "/" + directory
+	//create the repo directory
+	_ = os.Mkdir(repo.Dir, os.ModePerm)
+	return repo
+}
+
+func (repo *Repo) StorePad(link string, directory string, title string) (string, error) {
+	path, err := repo.GetPad(link, "md", directory, title)
+	if err != nil {
+		color.Red(err.Error())
+		return "", err
+	}
+
+	hash, err := IpfsAdd(path)
+	if err != nil {
+		color.Red(err.Error())
+		return hash, err
+	}
+
+	err = AddLineToFile(path, "IPFS hash of this document: "+hash)
+	if err != nil {
+		color.Red(err.Error())
+		return hash, err
+	}
+	// TODO
+	// err = repo.GitUpdate("update commit")
+	// if err != nil {
+	// 	color.Red(err.Error())
+	// 	return hash, err
+	// }
+
+	return hash, nil
+}

padArchiver_test.go

@@ -0,0 +1,22 @@
+package padArchiver
+
+import (
+	"testing"
+
+	"github.com/fatih/color"
+)
+
+const checkIcon = "\xE2\x9C\x94 "
+
+func TestAddPad(t *testing.T) {
+	color.Blue("TestAddPad")
+	repo := OpenRepo("Repo01")
+	_, err := repo.StorePad("http://board.net/p/pad1", "Group1", "pad1")
+
+	if err == nil {
+		color.Green(checkIcon + "checked AddPad")
+	} else {
+		color.Red(err.Error())
+		t.Errorf("Error AddPad")
+	}
+}