temp

Dockerfile

@@ -1,7 +1,21 @@
-FROM golang:1.16.4-alpine
+FROM golang:1.16-alpine AS pre
 
-ADD . /build
-WORKDIR /build/app
+RUN apk --no-cache add gcc libc-dev
 
-RUN go build -o main .
-CMD ["/build/app/main"]
+WORKDIR /build
+COPY . .
+
+RUN go build -o main ./cmd
+#RUN CGO_ENABLED=1 go test -v -race -timeout 30s ./...
+
+ENV CGO_ENABLED=0
+
+
+FROM alpine:3.13 AS bin
+
+RUN apk --no-cache add bash
+
+WORKDIR /app
+COPY --from=pre /build .
+
+CMD ["/app/main"]

Makefile

@@ -11,10 +11,16 @@ lint-yaml:
 	docker pull ${REGISTRY}/${YAMLLINT_TAG}
 	$(LINT-YAML-CMD) deployments/docker-compose.yml
 
+DOCKER-COMPOSE-CMD=docker-compose --file deployments/docker-compose.yml --project-name news-aggregator-back
+
+test:
+	go build -o main ./cmd
+	go test -v -race -timeout 30s ./...
+
 restart:
-	docker-compose --file deployments/docker-compose.yml --project-name news-aggregator-back build
-	docker-compose --file deployments/docker-compose.yml --project-name news-aggregator-back stop
-	docker-compose --file deployments/docker-compose.yml --project-name news-aggregator-back up -d
+	$(DOCKER-COMPOSE-CMD) build
+	$(DOCKER-COMPOSE-CMD) stop
+	$(DOCKER-COMPOSE-CMD) up -d
 
 cleanup:
 	docker system prune --all --force

app/main.go

@@ -1,72 +0,0 @@
-package main
-
-import (
-    "encoding/json"
-    "fmt"
-    "io/ioutil"
-    "log"
-    "net/http"
-    "time"
-)
-
-
-type Podcasts struct {
-    URL        string    `json:"url"`
-    Title      string    `json:"title"`
-    Date       time.Time `json:"date"`
-    Categories []string  `json:"categories"`
-    Image      string    `json:"image,omitempty"`
-    FileName   string    `json:"file_name,omitempty"`
-    Body       string    `json:"body"`
-    ShowNotes  string    `json:"show_notes,omitempty"`
-    AudioUrl   string    `json:"audio_url"`
-    TimeLabels []TimeLabels
-    ShowNum    int       `json:"show_num,omitempty"`
-}
-
-type TimeLabels struct {
-    Topic      string    `json:"topic,omitempty"`
-    Time       time.Time `json:"time,omitempty"`
-    Duration   int       `json:"duration,omitempty"`
-}
-
-
-func main() {
-    var lines []string
-    fmt.Println(getRadioTThemes(1))
-    for _, podcast := range getRadioTThemes(5) {
-        lines = append(lines, fmt.Sprintf("[%s](%s) %s\n %s\n\n",
-            podcast.Title,
-            podcast.URL,
-            podcast.Date.Format("2006-01-02"),
-            podcast.Body,
-        ))
-    }
-
-    http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-        fmt.Fprintf(w, "%s", lines)
-    })
-
-    log.Fatal(http.ListenAndServe(":7070", nil))
-}
-
-func getRadioTThemes(numPodcasts int) []Podcasts {
-    var url string
-    url = fmt.Sprintf("https://radio-t.com/site-api/last/%d?categories=podcast", numPodcasts)
-    response, err := http.Get(url)
-
-    if err != nil {
-        panic(err.Error())
-    }
-
-    body, err := ioutil.ReadAll(response.Body)
-
-    if err != nil {
-        panic(err.Error())
-    }
-
-    var podcasts []Podcasts
-    json.Unmarshal(body, &podcasts)
-
-    return podcasts
-}

cmd/main.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"gitea.fedy95.com/dev/news-aggregator-back/internal/app/apiserver"
+	"log"
+	//"gitea.fedy95.com/dev/news-aggregator-back/internal/app/apiserver"
+	//"log"
+)
+
+func main() {
+	s := apiserver.New()
+	if err := s.Start(); err != nil {
+		log.Fatal(err)
+	}
+}

data/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

deployments/docker-compose.yml

@@ -1,14 +1,28 @@
 ---
-version: "3.8"
+version: "3"
 
 services:
-    news-aggregator-back:
-        build:
-            context: ../
-        restart: always
-        container_name: news-aggregator-back
-        hostname: news-aggregator-back
+  news-aggregator-back:
+    build:
+      context: ../
+      target: bin
+    restart: on-failure
+    container_name: news-aggregator-back
+    hostname: news-aggregator-back
 
-        ports:
-            - "7070:7070"
+    env_file: env/news-aggregator-back.env
+    volumes:
+      - ../data:/app/data
+    ports:
+      - "7070:7070"
+#  postgresql:
+#    image: postgres:13.1-alpine
+#    container_name: postgresql
+#    hostname: postgresql
+#
+#    env_file: env/postgres.env
+#    expose:
+#      - "5432"
+#    volumes:
+#      - ./etc/postgresql/var/lib/postgresql/data:/var/lib/postgresql/data
 ...

deployments/env/postgres.env

@@ -0,0 +1,3 @@
+POSTGRES_DB=news-aggregator-back
+POSTGRES_USER=fedy95
+POSTGRES_PASSWORD=fedy95

deployments/env/postgres.env-dist

@@ -0,0 +1,3 @@
+POSTGRES_DB=
+POSTGRES_USER=
+POSTGRES_PASSWORD=

deployments/etc/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

docs/Dockerfile

@@ -0,0 +1,11 @@
+FROM openjdk:14-jdk-alpine3.10
+
+ENV PLANTUML_VERSION=1.2020.9
+ENV LANG en_US.UTF-8
+
+RUN \
+  apk add --no-cache graphviz wget ca-certificates ttf-dejavu fontconfig && \
+  wget "http://downloads.sourceforge.net/project/plantuml/${PLANTUML_VERSION}/plantuml.${PLANTUML_VERSION}.jar" -O plantuml.jar && \
+  apk del wget ca-certificates
+RUN ["java", "-Djava.awt.headless=true", "-jar", "plantuml.jar", "-version"]
+RUN ["dot", "-version"]

docs/README.md

@@ -0,0 +1 @@
+https://plantuml.com/

docs/build.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# shellcheck disable=SC2046
+
+docker build
+
+docker run --rm -v $(pwd)/data:/data -u $(id -u) --entrypoint ./data/generate.sh $CONTAINER

docs/puml/database.puml

@@ -0,0 +1,11 @@
+@startuml
+'https://plantuml.com/sequence-diagram
+
+autonumber
+
+Alice -> Bob: Authentication Request
+Bob --> Alice: Authentication Response
+
+Alice -> Bob: Another authentication Request
+Alice <-- Bob: another authentication Response
+@enduml

docs/puml/general.puml

@@ -0,0 +1,38 @@
+@startuml
+
+actor user
+participant news_aggregator_back
+database db
+participant source_1
+participant source_2
+
+activate user
+
+user -> news_aggregator_back ++  : /general
+  news_aggregator_back -> db ++ : GET
+  return response
+
+  group db records is outdated
+    news_aggregator_back -> source_1 ++ : GET
+    return response
+    news_aggregator_back -> db ++ : POST
+    return response
+
+    news_aggregator_back -> source_2 ++ : GET
+    return response
+    news_aggregator_back -> db ++ : POST
+    return response
+  end
+
+return response
+
+@enduml
+
+'request
+'
+'for sources as source
+'  check db
+'  if !updated
+'    update source
+'
+'show

go.mod

@@ -1,3 +1,11 @@
 module gitea.fedy95.com/dev/news-aggregator-back
 
 go 1.16
+
+require (
+	//github.com/codegangsta/cli v1.20.0 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/lib/pq v1.10.1 // indirect
+	github.com/mmcdole/gofeed v1.1.3 // indirect
+	github.com/sirupsen/logrus v1.8.1
+)

go.sum

@@ -0,0 +1,51 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo=
+github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o=
+github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/app/apiserver/apiserver.go

@@ -0,0 +1,97 @@
+package apiserver
+
+import (
+	"encoding/json"
+	"fmt"
+	"gitea.fedy95.com/dev/news-aggregator-back/internal/app/store"
+	"github.com/gorilla/mux"
+	"github.com/mmcdole/gofeed"
+	"github.com/sirupsen/logrus"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+type APIServer struct {
+	logger *logrus.Logger
+	router *mux.Router
+	store  *store.Store
+}
+
+func New() *APIServer {
+	return &APIServer{
+		logger: logrus.New(),
+		router: mux.NewRouter(),
+	}
+}
+
+func (srv *APIServer) Start() error {
+	if err := srv.configureLogger(); err != nil {
+		return err
+	}
+
+	srv.configureRouter()
+
+	srv.logger.Info("starting apiserver")
+	return http.ListenAndServe(":7070", srv.router) //TODO mv to env
+}
+
+func (srv *APIServer) configureLogger() error {
+	level, err := logrus.ParseLevel("debug") //TODO mv to env
+	if err != nil {
+		return err
+	}
+
+	srv.logger.SetLevel(level)
+
+	return nil
+}
+
+func (srv *APIServer) configureRouter() {
+	srv.router.HandleFunc("/test", srv.handlerTest())
+}
+
+func (srv *APIServer) handlerTest() http.HandlerFunc {
+	return func(writer http.ResponseWriter, request *http.Request) {
+		var rssSources [5]string
+		rssSources[0] = "https://www.opennet.ru/opennews/opennews_all_utf.rss"
+		rssSources[1] = "http://feeds.rucast.net/radio-t"
+		rssSources[2] = "http://habr.ru/rss/hubs/all"
+		rssSources[3] = "https://servernews.ru/news/rss"
+		rssSources[4] = "https://servernews.ru/rss"
+
+		writer.Header().Set("Content-Type", "text/html; charset=utf-8")
+
+		currentTime := time.Now()
+		for _, rssElement := range rssSources {
+			fp := gofeed.NewParser()
+			feed, _ := fp.ParseURL(rssElement)
+
+			file, _ := json.MarshalIndent(feed, "", " ")
+
+			var filename string
+			filename = fmt.Sprintf("data/%s_%s", currentTime.Format("2006.01.02 15:04:05"), feed.Title)
+			srv.logger.Info(fmt.Sprintf("write filename %s", filename))
+
+			ioutil.WriteFile(filename, file, 0644)
+
+			fmt.Fprintf(writer, "<h3>%s</h3><ul>\n", filename)
+
+			for _, element := range feed.Items {
+				fmt.Fprintf(writer, "<li><a href=\"%s\">%s</a></li>\n", element.Link, element.Title)
+			}
+			fmt.Fprintf(writer, "</ul>\n")
+		}
+	}
+}
+
+//func (srv *APIServer) configureStore() error {
+//	st := store.New(srv.config.Store)
+//	if err = st.Open(); err != nil {
+//		return err
+//	}
+//
+//  srv.store = st
+//
+//	return nil
+//}

internal/app/apiserver/apiserver_test.go

@@ -0,0 +1,16 @@
+package apiserver
+
+import (
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestAPIServer_HandleHello(t *testing.T) {
+	srv := New()
+	rec := httptest.NewRecorder()
+	req, _ := http.NewRequest(http.MethodGet, "/test", nil)
+	srv.handlerTest().ServeHTTP(rec, req)
+	assert.Equal(t, rec.Body.String(), "<h1>Title</h1><div>Body</div>")
+}

internal/app/store/store.go

@@ -0,0 +1,34 @@
+package store
+
+import (
+	"database/sql"
+
+	_ "github.com/lib/pq"
+)
+
+type Store struct {
+	db *sql.DB
+}
+
+func (s *Store) Open() error {
+	connStr := "postgres://fedy95:fedy95@postgresql/fedy95?sslmode=verify-full" //TODO mv to env
+	db, err := sql.Open("postgres", connStr)
+	if err != nil {
+		return err
+	}
+
+	if err := db.Ping(); err != nil {
+		return err
+	}
+
+	s.db = db
+
+	return nil
+}
+
+func (s *Store) Close() {
+	err := s.db.Close()
+	if err != nil {
+		return
+	}
+}