rpuzonas/go-rss-aggregator
1
0
Fork 0
Code

add showing posts that user is following

Browse Source
This commit is contained in:
Rokas Puzonas 2023-09-09 15:57:49 +03:00
parent bf17977dd8
commit 57b5a1b27b
16 changed files with 336 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
handler_feed.go
View File

@ -12,7 +12,7 @@ import (
func (apiCfg *apiConfig) handlerCreateFeed(w http.ResponseWriter, r *http.Request, user User) {
type parameters struct {
Name string `json:"name"`
URL string `json:"url"`
URL string `json:"url"`
}
decoder := json.NewDecoder(r.Body)
@ -27,9 +27,9 @@ func (apiCfg *apiConfig) handlerCreateFeed(w http.ResponseWriter, r *http.Reques
feed, err := apiCfg.DB.CreateFeed(r.Context(), database.CreateFeedParams{
UpdatedAt: now,
CreatedAt: now,
Name: params.Name,
Url: params.URL,
UserID: user.ID,
Name: params.Name,
Url: params.URL,
UserID: user.ID,
})
if err != nil {
respondWithError(w, 400, fmt.Sprintf("Couldn't create user: %v", err))

9
handler_feed_follows.go
View File

@ -11,7 +11,6 @@ import (
"github.com/go-chi/chi"
)
func (apiCfg *apiConfig) handlerFollowFeed(w http.ResponseWriter, r *http.Request, user User) {
type parameters struct {
FeedID int64 `json:"feed_id"`
@ -29,8 +28,8 @@ func (apiCfg *apiConfig) handlerFollowFeed(w http.ResponseWriter, r *http.Reques
feedFollow, err := apiCfg.DB.CreateFeedFollow(r.Context(), database.CreateFeedFollowParams{
UpdatedAt: now,
CreatedAt: now,
UserID: user.ID,
FeedID: params.FeedID,
UserID: user.ID,
FeedID: params.FeedID,
})
if err != nil {
respondWithError(w, 400, fmt.Sprintf("Couldn't follow feed: %v", err))
@ -51,7 +50,7 @@ func (apiCfg *apiConfig) handlerGetFeedFollows(w http.ResponseWriter, r *http.Re
}
func (apiCfg *apiConfig) handlerDeleteFeedFollow(w http.ResponseWriter, r *http.Request, user User) {
feedFollowIDStr := chi.URLParam(r, "feedFollowID");
feedFollowIDStr := chi.URLParam(r, "feedFollowID")
feedFollowID, err := strconv.ParseInt(feedFollowIDStr, 10, 64)
if err != nil {
respondWithError(w, 400, fmt.Sprintf("Failed to parse feed id: %v", err))
@ -60,7 +59,7 @@ func (apiCfg *apiConfig) handlerDeleteFeedFollow(w http.ResponseWriter, r *http.
err = apiCfg.DB.DeleteFeedFollow(r.Context(), database.DeleteFeedFollowParams{
UserID: user.ID,
ID: feedFollowID,
ID: feedFollowID,
})
if err != nil {
respondWithError(w, 400, fmt.Sprintf("Failed to delete feed follow: %v", err))

15
handler_user.go
View File

@ -26,7 +26,7 @@ func (apiCfg *apiConfig) handlerCreateUser(w http.ResponseWriter, r *http.Reques
user, err := apiCfg.DB.CreateUser(r.Context(), database.CreateUserParams{
UpdatedAt: now,
CreatedAt: now,
Name: params.Name,
Name: params.Name,
})
if err != nil {
respondWithError(w, 400, fmt.Sprintf("Couldn't create user: %v", err))
@ -39,3 +39,16 @@ func (apiCfg *apiConfig) handlerCreateUser(w http.ResponseWriter, r *http.Reques
func (apiCfg *apiConfig) handlerGetUser(w http.ResponseWriter, r *http.Request, user User) {
respondWithJSON(w, 200, user)
}
func (apiCfg *apiConfig) handlerGetPostsForUser(w http.ResponseWriter, r *http.Request, user User) {
posts, err := apiCfg.DB.GetPostsForUser(r.Context(), database.GetPostsForUserParams{
UserID: user.ID,
Limit: 10,
})
if err != nil {
respondWithError(w, 400, fmt.Sprintf("Failed to list posts for user: %v", err))
return
}
respondWithJSON(w, 200, databasePostsToPosts(posts))
}

6
http-requests/create-feed.http
View File

@ -1,8 +1,8 @@
POST http://localhost:8080/v1/feeds
Content-Type: application/json
Authorization: ApiKey aec0a1cbc6658af07e0d3ed67079a46928744b7787170a913abea4c68e9af702bbf2593bb8283b1574cf7b2e0d60df1abbd3e619c80e483d666f219ea71c72374e66b372a173e1407087c5cdeecf7dc66f34cdf59a0b6c448a86b6954ffc2693185550cd203625c8fa1eb0388e0834ee70cecec0dc199197a6e7229d28659fb6929c8410d74f0336a73ab62be5486451a270bcdbc37b0f73cad6654bc057fe28754e151330bbbc22c9e8d645015b81f035f95ec5b21b064965b65123f3a0705df4ab83a963d6a3353226cb7e9eeecdfb9631299e3068f9e28220e58bd7a69389a8697bee67501017cbde6fb2c809b136496533f40efed60be51e2b44899b8ade
Authorization: ApiKey 62fc6c4a02ce7f9d3af346550a37ac3f9f881f6a83987bae3b0a5cc951fc497b57a25e24cbc4ef9efa3e1192baca972e24029848bf9ed675784e36579eb30463b063a3547fe180be19d16525d73bb76f6c22aad4199699607e211e208ba5f316de6aafde53adec29f1be178f3d0d38aba11feb0b9e2687f2966f92f992b7707b2aa926067233749d0b4e18141a8f6ca18fad4d5458930a332040fbc2d9970cd7b9b53dd4220010c8c5759924a003afa28bd4cd6d96cb4f4c5c1efdcda8831f74f419cd4375915e4228be70a25afdb3d1dd46c735f2d9cf1569ed10298eff54c73ccda06c5763437537baaa68a9232e2c2ff3118691a6a2d4cccdf9cf60120d21
{
"name": "Joe's feed",
"url": "http://example.com/feed.xml"
"name": "Lane's blog",
"url": "https://www.wagslane.dev/index.xml"
}

4
http-requests/follow-feed.http
View File

@ -1,7 +1,7 @@
POST http://localhost:8080/v1/feed-follows
Content-Type: application/json
Authorization: ApiKey aec0a1cbc6658af07e0d3ed67079a46928744b7787170a913abea4c68e9af702bbf2593bb8283b1574cf7b2e0d60df1abbd3e619c80e483d666f219ea71c72374e66b372a173e1407087c5cdeecf7dc66f34cdf59a0b6c448a86b6954ffc2693185550cd203625c8fa1eb0388e0834ee70cecec0dc199197a6e7229d28659fb6929c8410d74f0336a73ab62be5486451a270bcdbc37b0f73cad6654bc057fe28754e151330bbbc22c9e8d645015b81f035f95ec5b21b064965b65123f3a0705df4ab83a963d6a3353226cb7e9eeecdfb9631299e3068f9e28220e58bd7a69389a8697bee67501017cbde6fb2c809b136496533f40efed60be51e2b44899b8ade
Authorization: ApiKey 62fc6c4a02ce7f9d3af346550a37ac3f9f881f6a83987bae3b0a5cc951fc497b57a25e24cbc4ef9efa3e1192baca972e24029848bf9ed675784e36579eb30463b063a3547fe180be19d16525d73bb76f6c22aad4199699607e211e208ba5f316de6aafde53adec29f1be178f3d0d38aba11feb0b9e2687f2966f92f992b7707b2aa926067233749d0b4e18141a8f6ca18fad4d5458930a332040fbc2d9970cd7b9b53dd4220010c8c5759924a003afa28bd4cd6d96cb4f4c5c1efdcda8831f74f419cd4375915e4228be70a25afdb3d1dd46c735f2d9cf1569ed10298eff54c73ccda06c5763437537baaa68a9232e2c2ff3118691a6a2d4cccdf9cf60120d21
{
"feed_id": 4
"feed_id": 1
}

3
http-requests/get-posts-for-user.http Normal file
View File

@ -0,0 +1,3 @@
GET http://localhost:8080/v1/user/posts
Content-Type: application/json
Authorization: ApiKey 62fc6c4a02ce7f9d3af346550a37ac3f9f881f6a83987bae3b0a5cc951fc497b57a25e24cbc4ef9efa3e1192baca972e24029848bf9ed675784e36579eb30463b063a3547fe180be19d16525d73bb76f6c22aad4199699607e211e208ba5f316de6aafde53adec29f1be178f3d0d38aba11feb0b9e2687f2966f92f992b7707b2aa926067233749d0b4e18141a8f6ca18fad4d5458930a332040fbc2d9970cd7b9b53dd4220010c8c5759924a003afa28bd4cd6d96cb4f4c5c1efdcda8831f74f419cd4375915e4228be70a25afdb3d1dd46c735f2d9cf1569ed10298eff54c73ccda06c5763437537baaa68a9232e2c2ff3118691a6a2d4cccdf9cf60120d21

11
internal/database/models.go
View File

@ -27,6 +27,17 @@ type FeedFollow struct {
FeedID int64
}
type Post struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
Title string
Description sql.NullString
PublishedAt time.Time
Url string
FeedID int64
}
type User struct {
ID int64
CreatedAt time.Time

140
internal/database/posts.sql.go Normal file
View File

@ -0,0 +1,140 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.21.0
// source: posts.sql
package database
import (
"context"
"database/sql"
"time"
)
const createPost = `-- name: CreatePost :one
INSERT INTO posts (
created_at,
updated_at,
title,
description,
published_at,
url,
feed_id)
VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING id, created_at, updated_at, title, description, published_at, url, feed_id
`
type CreatePostParams struct {
CreatedAt time.Time
UpdatedAt time.Time
Title string
Description sql.NullString
PublishedAt time.Time
Url string
FeedID int64
}
func (q *Queries) CreatePost(ctx context.Context, arg CreatePostParams) (Post, error) {
row := q.db.QueryRowContext(ctx, createPost,
arg.CreatedAt,
arg.UpdatedAt,
arg.Title,
arg.Description,
arg.PublishedAt,
arg.Url,
arg.FeedID,
)
var i Post
err := row.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.Title,
&i.Description,
&i.PublishedAt,
&i.Url,
&i.FeedID,
)
return i, err
}
const getPosts = `-- name: GetPosts :many
SELECT id, created_at, updated_at, title, description, published_at, url, feed_id FROM posts
`
func (q *Queries) GetPosts(ctx context.Context) ([]Post, error) {
rows, err := q.db.QueryContext(ctx, getPosts)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Post
for rows.Next() {
var i Post
if err := rows.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.Title,
&i.Description,
&i.PublishedAt,
&i.Url,
&i.FeedID,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getPostsForUser = `-- name: GetPostsForUser :many
SELECT posts.id, posts.created_at, posts.updated_at, posts.title, posts.description, posts.published_at, posts.url, posts.feed_id FROM posts
JOIN feed_follows ON posts.feed_id = feed_follows.id
WHERE feed_follows.user_id = ?
ORDER BY posts.published_at DESC
LIMIT ?
`
type GetPostsForUserParams struct {
UserID int64
Limit int64
}
func (q *Queries) GetPostsForUser(ctx context.Context, arg GetPostsForUserParams) ([]Post, error) {
rows, err := q.db.QueryContext(ctx, getPostsForUser, arg.UserID, arg.Limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Post
for rows.Next() {
var i Post
if err := rows.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.Title,
&i.Description,
&i.PublishedAt,
&i.Url,
&i.FeedID,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}

8
json.go
View File

@ -7,7 +7,7 @@ import (
)
func respondWithError(w http.ResponseWriter, code int, msg string) {
if code / 100 == 5 {
if code/100 == 5 {
log.Println("Responding with 5XX error:", msg)
}
@ -15,7 +15,7 @@ func respondWithError(w http.ResponseWriter, code int, msg string) {
Error string `json:"error"`
}
respondWithJSON(w, code, errorResponse{ Error: msg })
respondWithJSON(w, code, errorResponse{Error: msg})
}
func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) {
@ -26,7 +26,7 @@ func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) {
return
}
w.Header().Add("Content-Type", "application/json");
w.WriteHeader(code);
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(code)
w.Write(dat)
}

21
main.go
View File

@ -29,7 +29,7 @@ func assert_getenv(name string) string {
}
func createV1Router(cfg *apiConfig) chi.Router {
v1Router := chi.NewRouter();
v1Router := chi.NewRouter()
v1Router.Get("/healthz", handlerReadiness)
v1Router.Get("/err", handleErr)
@ -37,6 +37,7 @@ func createV1Router(cfg *apiConfig) chi.Router {
v1Router.Post("/user", cfg.handlerCreateUser)
v1Router.Get("/user", cfg.middlewareAuth(cfg.handlerGetUser))
v1Router.Get("/user/feeds", cfg.middlewareAuth(cfg.handlerGetCreatedFeeds))
v1Router.Get("/user/posts", cfg.middlewareAuth(cfg.handlerGetPostsForUser))
v1Router.Post("/feeds", cfg.middlewareAuth(cfg.handlerCreateFeed))
v1Router.Get("/feeds", cfg.handlerGetAllFeeds)
@ -61,29 +62,29 @@ func main() {
defer conn.Close()
db := database.New(conn)
apiCfg := apiConfig{ DB: db }
apiCfg := apiConfig{DB: db}
router := chi.NewRouter()
router.Use(cors.Handler(cors.Options{
AllowedOrigins: []string{"https://*", "http://*"},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"*"},
ExposedHeaders: []string{"Link"},
AllowedOrigins: []string{"https://*", "http://*"},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"*"},
ExposedHeaders: []string{"Link"},
AllowCredentials: false,
MaxAge: 300,
MaxAge: 300,
}))
router.Mount("/v1", createV1Router(&apiCfg))
srv := &http.Server{
Handler: router,
Addr: ":" + portStr,
};
Addr: ":" + portStr,
}
go startScraping(db, 10, time.Minute)
fmt.Printf("Listening on port: %s\n", portStr)
err = srv.ListenAndServe();
err = srv.ListenAndServe()
if err != nil {
log.Fatal(err)
}

2
middleware_auth.go
View File

@ -18,7 +18,7 @@ func (cfg *apiConfig) middlewareAuth(handler authedHandler) http.HandlerFunc {
return
}
user, err := cfg.DB.GetUserByAPIKey(r.Context(), sql.NullString{ String: apiKey, Valid: true })
user, err := cfg.DB.GetUserByAPIKey(r.Context(), sql.NullString{String: apiKey, Valid: true})
if err != nil {
respondWithError(w, 400, fmt.Sprintf("Couldn't get user: %v", err))
return

85
models.go
View File

@ -7,48 +7,59 @@ import (
)
type User struct {
ID int64 `json:"id"`
ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Name string `json:"name"`
ApiKey string `json:"api_key"`
Name string `json:"name"`
ApiKey string `json:"api_key"`
}
type Feed struct {
ID int64 `json:"id"`
ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Name string `json:"name"`
Url string `json:"url"`
UserID int64 `json:"user_id"`
Name string `json:"name"`
Url string `json:"url"`
UserID int64 `json:"user_id"`
}
type FeedFollow struct {
ID int64 `json:"id"`
ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
UserID int64 `json:"user_id"`
FeedID int64 `json:"feed_id"`
UserID int64 `json:"user_id"`
FeedID int64 `json:"feed_id"`
}
type Post struct {
ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Title string `json:"title"`
Description *string `json:"description"`
PublishedAt time.Time `json:"published_at"`
URL string `json:"url"`
FeedID int64 `json:"feed_id"`
}
func databaseUserToUser(dbUser database.User) User {
return User{
ID: dbUser.ID,
ID: dbUser.ID,
CreatedAt: dbUser.CreatedAt,
UpdatedAt: dbUser.UpdatedAt,
Name: dbUser.Name,
ApiKey: dbUser.ApiKey.String,
Name: dbUser.Name,
ApiKey: dbUser.ApiKey.String,
}
}
func databaseFeedToFeed(dbFeed database.Feed) Feed {
return Feed{
ID: dbFeed.ID,
ID: dbFeed.ID,
CreatedAt: dbFeed.CreatedAt,
UpdatedAt: dbFeed.UpdatedAt,
Name: dbFeed.Name,
Url: dbFeed.Url,
UserID: dbFeed.UserID,
Name: dbFeed.Name,
Url: dbFeed.Url,
UserID: dbFeed.UserID,
}
}
@ -62,18 +73,44 @@ func databaseFeedsToFeeds(dbFeeds []database.Feed) []Feed {
func databaseFeedFollowToFeedFollow(dbFeedFollow database.FeedFollow) FeedFollow {
return FeedFollow{
ID: dbFeedFollow.ID,
ID: dbFeedFollow.ID,
CreatedAt: dbFeedFollow.CreatedAt,
UpdatedAt: dbFeedFollow.UpdatedAt,
UserID: dbFeedFollow.UserID,
FeedID: dbFeedFollow.FeedID,
UserID: dbFeedFollow.UserID,
FeedID: dbFeedFollow.FeedID,
}
}
func databaseFolllowFeedsToFollowFeeds(dbFeedFollows []database.FeedFollow) []FeedFollow {
feeds := make([]FeedFollow, len(dbFeedFollows))
for i, dbFeed := range dbFeedFollows {
feeds[i] = databaseFeedFollowToFeedFollow(dbFeed)
feedFollows := make([]FeedFollow, len(dbFeedFollows))
for i, dbFeedFollow := range dbFeedFollows {
feedFollows[i] = databaseFeedFollowToFeedFollow(dbFeedFollow)
}
return feeds
return feedFollows
}
func databasePostToPost(dbPost database.Post) Post {
var description *string = nil
if dbPost.Description.Valid {
description = &dbPost.Description.String
}
return Post{
ID: dbPost.ID,
CreatedAt: dbPost.CreatedAt,
UpdatedAt: dbPost.UpdatedAt,
Title: dbPost.Title,
Description: description,
PublishedAt: dbPost.PublishedAt,
URL: dbPost.Url,
FeedID: dbPost.FeedID,
}
}
func databasePostsToPosts(dbPosts []database.Post) []Post {
posts := make([]Post, len(dbPosts))
for i, dbPost := range dbPosts {
posts[i] = databasePostToPost(dbPost)
}
return posts
}

20
rss.go
View File

@ -9,23 +9,23 @@ import (
type RSSFeed struct {
Channel struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
Language string `xml:"language"`
Items []RSSItem `xml:"item"`
} `xml:"channel"`;
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
Language string `xml:"language"`
Items []RSSItem `xml:"item"`
} `xml:"channel"`
}
type RSSItem struct {
Title string `xml:"title"`
Link string `xml:"link"`
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
PubDate string `xml:"pubDate"`
PubDate string `xml:"pubDate"`
}
func urlToFeed(url string) (RSSFeed, error) {
httpClient := http.Client{ Timeout: 10 * time.Second }
httpClient := http.Client{Timeout: 10 * time.Second}
resp, err := httpClient.Get(url)
if err != nil {

35
scraper.go
View File

@ -2,7 +2,9 @@ package main
import (
"context"
"database/sql"
"log"
"strings"
"sync"
"time"
@ -36,7 +38,9 @@ func startScraping(
func scrapeFeed(wg *sync.WaitGroup, db *database.Queries, feed database.Feed) {
defer wg.Done()
_, err := db.MarkFeedAsFetched(context.Background(), feed.ID)
ctx := context.Background()
_, err := db.MarkFeedAsFetched(ctx, feed.ID)
if err != nil {
log.Println("Error makrking feed as fetched:", err)
return
@ -49,7 +53,34 @@ func scrapeFeed(wg *sync.WaitGroup, db *database.Queries, feed database.Feed) {
}
for _, item := range rssFeed.Channel.Items {
log.Println("Found post:", item.Title, "on feed", feed.Name)
description := sql.NullString{}
if item.Description != "" {
description.String = item.Description
description.Valid = true
}
publishedAt, err := time.Parse(time.RFC1123Z, item.PubDate)
if err != nil {
log.Printf("Couldn't parse date %v with err %v\n", item.PubDate, err)
continue
}
now := time.Now().UTC()
_, err = db.CreatePost(ctx, database.CreatePostParams{
UpdatedAt: now,
CreatedAt: now,
Title: item.Title,
Description: description,
PublishedAt: publishedAt,
Url: item.Link,
FeedID: feed.ID,
})
if err != nil {
if strings.Contains(err.Error(), "UNIQUE constraint failed: posts.url") {
continue
}
log.Println("Failed to add post:", err)
}
}
log.Printf("Feed %s collected, %v posts found", feed.Name, len(rssFeed.Channel.Items))
}

21
sql/queries/posts.sql Normal file
View File

@ -0,0 +1,21 @@
-- name: CreatePost :one
INSERT INTO posts (
created_at,
updated_at,
title,
description,
published_at,
url,
feed_id)
VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING *;
-- name: GetPosts :many
SELECT * FROM posts;
-- name: GetPostsForUser :many
SELECT posts.* FROM posts
JOIN feed_follows ON posts.feed_id = feed_follows.id
WHERE feed_follows.user_id = ?
ORDER BY posts.published_at DESC
LIMIT ?;

14
sql/schema/006_posts.sql Normal file
View File

@ -0,0 +1,14 @@
-- +goose Up
CREATE TABLE posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_at DATE NOT NULL,
updated_at DATE NOT NULL,
title TEXT NOT NULL,
description TEXT,
published_at DATE NOT NULL,
url TEXT NOT NULL UNIQUE,
feed_id INTEGER NOT NULL REFERENCES feeds(id) ON DELETE CASCADE
);
-- +goose Down
DROP TABLE posts;