1
0

add showing posts that user is following

This commit is contained in:
Rokas Puzonas 2023-09-09 15:57:49 +03:00
parent bf17977dd8
commit 57b5a1b27b
16 changed files with 336 additions and 66 deletions

View File

@ -12,7 +12,7 @@ import (
func (apiCfg *apiConfig) handlerCreateFeed(w http.ResponseWriter, r *http.Request, user User) { func (apiCfg *apiConfig) handlerCreateFeed(w http.ResponseWriter, r *http.Request, user User) {
type parameters struct { type parameters struct {
Name string `json:"name"` Name string `json:"name"`
URL string `json:"url"` URL string `json:"url"`
} }
decoder := json.NewDecoder(r.Body) decoder := json.NewDecoder(r.Body)
@ -27,9 +27,9 @@ func (apiCfg *apiConfig) handlerCreateFeed(w http.ResponseWriter, r *http.Reques
feed, err := apiCfg.DB.CreateFeed(r.Context(), database.CreateFeedParams{ feed, err := apiCfg.DB.CreateFeed(r.Context(), database.CreateFeedParams{
UpdatedAt: now, UpdatedAt: now,
CreatedAt: now, CreatedAt: now,
Name: params.Name, Name: params.Name,
Url: params.URL, Url: params.URL,
UserID: user.ID, UserID: user.ID,
}) })
if err != nil { if err != nil {
respondWithError(w, 400, fmt.Sprintf("Couldn't create user: %v", err)) respondWithError(w, 400, fmt.Sprintf("Couldn't create user: %v", err))

View File

@ -11,7 +11,6 @@ import (
"github.com/go-chi/chi" "github.com/go-chi/chi"
) )
func (apiCfg *apiConfig) handlerFollowFeed(w http.ResponseWriter, r *http.Request, user User) { func (apiCfg *apiConfig) handlerFollowFeed(w http.ResponseWriter, r *http.Request, user User) {
type parameters struct { type parameters struct {
FeedID int64 `json:"feed_id"` FeedID int64 `json:"feed_id"`
@ -29,8 +28,8 @@ func (apiCfg *apiConfig) handlerFollowFeed(w http.ResponseWriter, r *http.Reques
feedFollow, err := apiCfg.DB.CreateFeedFollow(r.Context(), database.CreateFeedFollowParams{ feedFollow, err := apiCfg.DB.CreateFeedFollow(r.Context(), database.CreateFeedFollowParams{
UpdatedAt: now, UpdatedAt: now,
CreatedAt: now, CreatedAt: now,
UserID: user.ID, UserID: user.ID,
FeedID: params.FeedID, FeedID: params.FeedID,
}) })
if err != nil { if err != nil {
respondWithError(w, 400, fmt.Sprintf("Couldn't follow feed: %v", err)) respondWithError(w, 400, fmt.Sprintf("Couldn't follow feed: %v", err))
@ -51,7 +50,7 @@ func (apiCfg *apiConfig) handlerGetFeedFollows(w http.ResponseWriter, r *http.Re
} }
func (apiCfg *apiConfig) handlerDeleteFeedFollow(w http.ResponseWriter, r *http.Request, user User) { func (apiCfg *apiConfig) handlerDeleteFeedFollow(w http.ResponseWriter, r *http.Request, user User) {
feedFollowIDStr := chi.URLParam(r, "feedFollowID"); feedFollowIDStr := chi.URLParam(r, "feedFollowID")
feedFollowID, err := strconv.ParseInt(feedFollowIDStr, 10, 64) feedFollowID, err := strconv.ParseInt(feedFollowIDStr, 10, 64)
if err != nil { if err != nil {
respondWithError(w, 400, fmt.Sprintf("Failed to parse feed id: %v", err)) respondWithError(w, 400, fmt.Sprintf("Failed to parse feed id: %v", err))
@ -60,7 +59,7 @@ func (apiCfg *apiConfig) handlerDeleteFeedFollow(w http.ResponseWriter, r *http.
err = apiCfg.DB.DeleteFeedFollow(r.Context(), database.DeleteFeedFollowParams{ err = apiCfg.DB.DeleteFeedFollow(r.Context(), database.DeleteFeedFollowParams{
UserID: user.ID, UserID: user.ID,
ID: feedFollowID, ID: feedFollowID,
}) })
if err != nil { if err != nil {
respondWithError(w, 400, fmt.Sprintf("Failed to delete feed follow: %v", err)) respondWithError(w, 400, fmt.Sprintf("Failed to delete feed follow: %v", err))

View File

@ -26,7 +26,7 @@ func (apiCfg *apiConfig) handlerCreateUser(w http.ResponseWriter, r *http.Reques
user, err := apiCfg.DB.CreateUser(r.Context(), database.CreateUserParams{ user, err := apiCfg.DB.CreateUser(r.Context(), database.CreateUserParams{
UpdatedAt: now, UpdatedAt: now,
CreatedAt: now, CreatedAt: now,
Name: params.Name, Name: params.Name,
}) })
if err != nil { if err != nil {
respondWithError(w, 400, fmt.Sprintf("Couldn't create user: %v", err)) respondWithError(w, 400, fmt.Sprintf("Couldn't create user: %v", err))
@ -39,3 +39,16 @@ func (apiCfg *apiConfig) handlerCreateUser(w http.ResponseWriter, r *http.Reques
func (apiCfg *apiConfig) handlerGetUser(w http.ResponseWriter, r *http.Request, user User) { func (apiCfg *apiConfig) handlerGetUser(w http.ResponseWriter, r *http.Request, user User) {
respondWithJSON(w, 200, user) respondWithJSON(w, 200, user)
} }
func (apiCfg *apiConfig) handlerGetPostsForUser(w http.ResponseWriter, r *http.Request, user User) {
posts, err := apiCfg.DB.GetPostsForUser(r.Context(), database.GetPostsForUserParams{
UserID: user.ID,
Limit: 10,
})
if err != nil {
respondWithError(w, 400, fmt.Sprintf("Failed to list posts for user: %v", err))
return
}
respondWithJSON(w, 200, databasePostsToPosts(posts))
}

View File

@ -1,8 +1,8 @@
POST http://localhost:8080/v1/feeds POST http://localhost:8080/v1/feeds
Content-Type: application/json Content-Type: application/json
Authorization: ApiKey aec0a1cbc6658af07e0d3ed67079a46928744b7787170a913abea4c68e9af702bbf2593bb8283b1574cf7b2e0d60df1abbd3e619c80e483d666f219ea71c72374e66b372a173e1407087c5cdeecf7dc66f34cdf59a0b6c448a86b6954ffc2693185550cd203625c8fa1eb0388e0834ee70cecec0dc199197a6e7229d28659fb6929c8410d74f0336a73ab62be5486451a270bcdbc37b0f73cad6654bc057fe28754e151330bbbc22c9e8d645015b81f035f95ec5b21b064965b65123f3a0705df4ab83a963d6a3353226cb7e9eeecdfb9631299e3068f9e28220e58bd7a69389a8697bee67501017cbde6fb2c809b136496533f40efed60be51e2b44899b8ade Authorization: ApiKey 62fc6c4a02ce7f9d3af346550a37ac3f9f881f6a83987bae3b0a5cc951fc497b57a25e24cbc4ef9efa3e1192baca972e24029848bf9ed675784e36579eb30463b063a3547fe180be19d16525d73bb76f6c22aad4199699607e211e208ba5f316de6aafde53adec29f1be178f3d0d38aba11feb0b9e2687f2966f92f992b7707b2aa926067233749d0b4e18141a8f6ca18fad4d5458930a332040fbc2d9970cd7b9b53dd4220010c8c5759924a003afa28bd4cd6d96cb4f4c5c1efdcda8831f74f419cd4375915e4228be70a25afdb3d1dd46c735f2d9cf1569ed10298eff54c73ccda06c5763437537baaa68a9232e2c2ff3118691a6a2d4cccdf9cf60120d21
{ {
"name": "Joe's feed", "name": "Lane's blog",
"url": "http://example.com/feed.xml" "url": "https://www.wagslane.dev/index.xml"
} }

View File

@ -1,7 +1,7 @@
POST http://localhost:8080/v1/feed-follows POST http://localhost:8080/v1/feed-follows
Content-Type: application/json Content-Type: application/json
Authorization: ApiKey aec0a1cbc6658af07e0d3ed67079a46928744b7787170a913abea4c68e9af702bbf2593bb8283b1574cf7b2e0d60df1abbd3e619c80e483d666f219ea71c72374e66b372a173e1407087c5cdeecf7dc66f34cdf59a0b6c448a86b6954ffc2693185550cd203625c8fa1eb0388e0834ee70cecec0dc199197a6e7229d28659fb6929c8410d74f0336a73ab62be5486451a270bcdbc37b0f73cad6654bc057fe28754e151330bbbc22c9e8d645015b81f035f95ec5b21b064965b65123f3a0705df4ab83a963d6a3353226cb7e9eeecdfb9631299e3068f9e28220e58bd7a69389a8697bee67501017cbde6fb2c809b136496533f40efed60be51e2b44899b8ade Authorization: ApiKey 62fc6c4a02ce7f9d3af346550a37ac3f9f881f6a83987bae3b0a5cc951fc497b57a25e24cbc4ef9efa3e1192baca972e24029848bf9ed675784e36579eb30463b063a3547fe180be19d16525d73bb76f6c22aad4199699607e211e208ba5f316de6aafde53adec29f1be178f3d0d38aba11feb0b9e2687f2966f92f992b7707b2aa926067233749d0b4e18141a8f6ca18fad4d5458930a332040fbc2d9970cd7b9b53dd4220010c8c5759924a003afa28bd4cd6d96cb4f4c5c1efdcda8831f74f419cd4375915e4228be70a25afdb3d1dd46c735f2d9cf1569ed10298eff54c73ccda06c5763437537baaa68a9232e2c2ff3118691a6a2d4cccdf9cf60120d21
{ {
"feed_id": 4 "feed_id": 1
} }

View File

@ -0,0 +1,3 @@
GET http://localhost:8080/v1/user/posts
Content-Type: application/json
Authorization: ApiKey 62fc6c4a02ce7f9d3af346550a37ac3f9f881f6a83987bae3b0a5cc951fc497b57a25e24cbc4ef9efa3e1192baca972e24029848bf9ed675784e36579eb30463b063a3547fe180be19d16525d73bb76f6c22aad4199699607e211e208ba5f316de6aafde53adec29f1be178f3d0d38aba11feb0b9e2687f2966f92f992b7707b2aa926067233749d0b4e18141a8f6ca18fad4d5458930a332040fbc2d9970cd7b9b53dd4220010c8c5759924a003afa28bd4cd6d96cb4f4c5c1efdcda8831f74f419cd4375915e4228be70a25afdb3d1dd46c735f2d9cf1569ed10298eff54c73ccda06c5763437537baaa68a9232e2c2ff3118691a6a2d4cccdf9cf60120d21

View File

@ -27,6 +27,17 @@ type FeedFollow struct {
FeedID int64 FeedID int64
} }
type Post struct {
ID int64
CreatedAt time.Time
UpdatedAt time.Time
Title string
Description sql.NullString
PublishedAt time.Time
Url string
FeedID int64
}
type User struct { type User struct {
ID int64 ID int64
CreatedAt time.Time CreatedAt time.Time

View File

@ -0,0 +1,140 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.21.0
// source: posts.sql
package database
import (
"context"
"database/sql"
"time"
)
const createPost = `-- name: CreatePost :one
INSERT INTO posts (
created_at,
updated_at,
title,
description,
published_at,
url,
feed_id)
VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING id, created_at, updated_at, title, description, published_at, url, feed_id
`
type CreatePostParams struct {
CreatedAt time.Time
UpdatedAt time.Time
Title string
Description sql.NullString
PublishedAt time.Time
Url string
FeedID int64
}
func (q *Queries) CreatePost(ctx context.Context, arg CreatePostParams) (Post, error) {
row := q.db.QueryRowContext(ctx, createPost,
arg.CreatedAt,
arg.UpdatedAt,
arg.Title,
arg.Description,
arg.PublishedAt,
arg.Url,
arg.FeedID,
)
var i Post
err := row.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.Title,
&i.Description,
&i.PublishedAt,
&i.Url,
&i.FeedID,
)
return i, err
}
const getPosts = `-- name: GetPosts :many
SELECT id, created_at, updated_at, title, description, published_at, url, feed_id FROM posts
`
func (q *Queries) GetPosts(ctx context.Context) ([]Post, error) {
rows, err := q.db.QueryContext(ctx, getPosts)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Post
for rows.Next() {
var i Post
if err := rows.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.Title,
&i.Description,
&i.PublishedAt,
&i.Url,
&i.FeedID,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getPostsForUser = `-- name: GetPostsForUser :many
SELECT posts.id, posts.created_at, posts.updated_at, posts.title, posts.description, posts.published_at, posts.url, posts.feed_id FROM posts
JOIN feed_follows ON posts.feed_id = feed_follows.id
WHERE feed_follows.user_id = ?
ORDER BY posts.published_at DESC
LIMIT ?
`
type GetPostsForUserParams struct {
UserID int64
Limit int64
}
func (q *Queries) GetPostsForUser(ctx context.Context, arg GetPostsForUserParams) ([]Post, error) {
rows, err := q.db.QueryContext(ctx, getPostsForUser, arg.UserID, arg.Limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Post
for rows.Next() {
var i Post
if err := rows.Scan(
&i.ID,
&i.CreatedAt,
&i.UpdatedAt,
&i.Title,
&i.Description,
&i.PublishedAt,
&i.Url,
&i.FeedID,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}

View File

@ -7,7 +7,7 @@ import (
) )
func respondWithError(w http.ResponseWriter, code int, msg string) { func respondWithError(w http.ResponseWriter, code int, msg string) {
if code / 100 == 5 { if code/100 == 5 {
log.Println("Responding with 5XX error:", msg) log.Println("Responding with 5XX error:", msg)
} }
@ -15,7 +15,7 @@ func respondWithError(w http.ResponseWriter, code int, msg string) {
Error string `json:"error"` Error string `json:"error"`
} }
respondWithJSON(w, code, errorResponse{ Error: msg }) respondWithJSON(w, code, errorResponse{Error: msg})
} }
func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) { func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) {
@ -26,7 +26,7 @@ func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) {
return return
} }
w.Header().Add("Content-Type", "application/json"); w.Header().Add("Content-Type", "application/json")
w.WriteHeader(code); w.WriteHeader(code)
w.Write(dat) w.Write(dat)
} }

21
main.go
View File

@ -29,7 +29,7 @@ func assert_getenv(name string) string {
} }
func createV1Router(cfg *apiConfig) chi.Router { func createV1Router(cfg *apiConfig) chi.Router {
v1Router := chi.NewRouter(); v1Router := chi.NewRouter()
v1Router.Get("/healthz", handlerReadiness) v1Router.Get("/healthz", handlerReadiness)
v1Router.Get("/err", handleErr) v1Router.Get("/err", handleErr)
@ -37,6 +37,7 @@ func createV1Router(cfg *apiConfig) chi.Router {
v1Router.Post("/user", cfg.handlerCreateUser) v1Router.Post("/user", cfg.handlerCreateUser)
v1Router.Get("/user", cfg.middlewareAuth(cfg.handlerGetUser)) v1Router.Get("/user", cfg.middlewareAuth(cfg.handlerGetUser))
v1Router.Get("/user/feeds", cfg.middlewareAuth(cfg.handlerGetCreatedFeeds)) v1Router.Get("/user/feeds", cfg.middlewareAuth(cfg.handlerGetCreatedFeeds))
v1Router.Get("/user/posts", cfg.middlewareAuth(cfg.handlerGetPostsForUser))
v1Router.Post("/feeds", cfg.middlewareAuth(cfg.handlerCreateFeed)) v1Router.Post("/feeds", cfg.middlewareAuth(cfg.handlerCreateFeed))
v1Router.Get("/feeds", cfg.handlerGetAllFeeds) v1Router.Get("/feeds", cfg.handlerGetAllFeeds)
@ -61,29 +62,29 @@ func main() {
defer conn.Close() defer conn.Close()
db := database.New(conn) db := database.New(conn)
apiCfg := apiConfig{ DB: db } apiCfg := apiConfig{DB: db}
router := chi.NewRouter() router := chi.NewRouter()
router.Use(cors.Handler(cors.Options{ router.Use(cors.Handler(cors.Options{
AllowedOrigins: []string{"https://*", "http://*"}, AllowedOrigins: []string{"https://*", "http://*"},
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
AllowedHeaders: []string{"*"}, AllowedHeaders: []string{"*"},
ExposedHeaders: []string{"Link"}, ExposedHeaders: []string{"Link"},
AllowCredentials: false, AllowCredentials: false,
MaxAge: 300, MaxAge: 300,
})) }))
router.Mount("/v1", createV1Router(&apiCfg)) router.Mount("/v1", createV1Router(&apiCfg))
srv := &http.Server{ srv := &http.Server{
Handler: router, Handler: router,
Addr: ":" + portStr, Addr: ":" + portStr,
}; }
go startScraping(db, 10, time.Minute) go startScraping(db, 10, time.Minute)
fmt.Printf("Listening on port: %s\n", portStr) fmt.Printf("Listening on port: %s\n", portStr)
err = srv.ListenAndServe(); err = srv.ListenAndServe()
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

View File

@ -18,7 +18,7 @@ func (cfg *apiConfig) middlewareAuth(handler authedHandler) http.HandlerFunc {
return return
} }
user, err := cfg.DB.GetUserByAPIKey(r.Context(), sql.NullString{ String: apiKey, Valid: true }) user, err := cfg.DB.GetUserByAPIKey(r.Context(), sql.NullString{String: apiKey, Valid: true})
if err != nil { if err != nil {
respondWithError(w, 400, fmt.Sprintf("Couldn't get user: %v", err)) respondWithError(w, 400, fmt.Sprintf("Couldn't get user: %v", err))
return return

View File

@ -7,48 +7,59 @@ import (
) )
type User struct { type User struct {
ID int64 `json:"id"` ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"` UpdatedAt time.Time `json:"updated_at"`
Name string `json:"name"` Name string `json:"name"`
ApiKey string `json:"api_key"` ApiKey string `json:"api_key"`
} }
type Feed struct { type Feed struct {
ID int64 `json:"id"` ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"` UpdatedAt time.Time `json:"updated_at"`
Name string `json:"name"` Name string `json:"name"`
Url string `json:"url"` Url string `json:"url"`
UserID int64 `json:"user_id"` UserID int64 `json:"user_id"`
} }
type FeedFollow struct { type FeedFollow struct {
ID int64 `json:"id"` ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"` UpdatedAt time.Time `json:"updated_at"`
UserID int64 `json:"user_id"` UserID int64 `json:"user_id"`
FeedID int64 `json:"feed_id"` FeedID int64 `json:"feed_id"`
}
type Post struct {
ID int64 `json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Title string `json:"title"`
Description *string `json:"description"`
PublishedAt time.Time `json:"published_at"`
URL string `json:"url"`
FeedID int64 `json:"feed_id"`
} }
func databaseUserToUser(dbUser database.User) User { func databaseUserToUser(dbUser database.User) User {
return User{ return User{
ID: dbUser.ID, ID: dbUser.ID,
CreatedAt: dbUser.CreatedAt, CreatedAt: dbUser.CreatedAt,
UpdatedAt: dbUser.UpdatedAt, UpdatedAt: dbUser.UpdatedAt,
Name: dbUser.Name, Name: dbUser.Name,
ApiKey: dbUser.ApiKey.String, ApiKey: dbUser.ApiKey.String,
} }
} }
func databaseFeedToFeed(dbFeed database.Feed) Feed { func databaseFeedToFeed(dbFeed database.Feed) Feed {
return Feed{ return Feed{
ID: dbFeed.ID, ID: dbFeed.ID,
CreatedAt: dbFeed.CreatedAt, CreatedAt: dbFeed.CreatedAt,
UpdatedAt: dbFeed.UpdatedAt, UpdatedAt: dbFeed.UpdatedAt,
Name: dbFeed.Name, Name: dbFeed.Name,
Url: dbFeed.Url, Url: dbFeed.Url,
UserID: dbFeed.UserID, UserID: dbFeed.UserID,
} }
} }
@ -62,18 +73,44 @@ func databaseFeedsToFeeds(dbFeeds []database.Feed) []Feed {
func databaseFeedFollowToFeedFollow(dbFeedFollow database.FeedFollow) FeedFollow { func databaseFeedFollowToFeedFollow(dbFeedFollow database.FeedFollow) FeedFollow {
return FeedFollow{ return FeedFollow{
ID: dbFeedFollow.ID, ID: dbFeedFollow.ID,
CreatedAt: dbFeedFollow.CreatedAt, CreatedAt: dbFeedFollow.CreatedAt,
UpdatedAt: dbFeedFollow.UpdatedAt, UpdatedAt: dbFeedFollow.UpdatedAt,
UserID: dbFeedFollow.UserID, UserID: dbFeedFollow.UserID,
FeedID: dbFeedFollow.FeedID, FeedID: dbFeedFollow.FeedID,
} }
} }
func databaseFolllowFeedsToFollowFeeds(dbFeedFollows []database.FeedFollow) []FeedFollow { func databaseFolllowFeedsToFollowFeeds(dbFeedFollows []database.FeedFollow) []FeedFollow {
feeds := make([]FeedFollow, len(dbFeedFollows)) feedFollows := make([]FeedFollow, len(dbFeedFollows))
for i, dbFeed := range dbFeedFollows { for i, dbFeedFollow := range dbFeedFollows {
feeds[i] = databaseFeedFollowToFeedFollow(dbFeed) feedFollows[i] = databaseFeedFollowToFeedFollow(dbFeedFollow)
} }
return feeds return feedFollows
}
func databasePostToPost(dbPost database.Post) Post {
var description *string = nil
if dbPost.Description.Valid {
description = &dbPost.Description.String
}
return Post{
ID: dbPost.ID,
CreatedAt: dbPost.CreatedAt,
UpdatedAt: dbPost.UpdatedAt,
Title: dbPost.Title,
Description: description,
PublishedAt: dbPost.PublishedAt,
URL: dbPost.Url,
FeedID: dbPost.FeedID,
}
}
func databasePostsToPosts(dbPosts []database.Post) []Post {
posts := make([]Post, len(dbPosts))
for i, dbPost := range dbPosts {
posts[i] = databasePostToPost(dbPost)
}
return posts
} }

20
rss.go
View File

@ -9,23 +9,23 @@ import (
type RSSFeed struct { type RSSFeed struct {
Channel struct { Channel struct {
Title string `xml:"title"` Title string `xml:"title"`
Link string `xml:"link"` Link string `xml:"link"`
Description string `xml:"description"` Description string `xml:"description"`
Language string `xml:"language"` Language string `xml:"language"`
Items []RSSItem `xml:"item"` Items []RSSItem `xml:"item"`
} `xml:"channel"`; } `xml:"channel"`
} }
type RSSItem struct { type RSSItem struct {
Title string `xml:"title"` Title string `xml:"title"`
Link string `xml:"link"` Link string `xml:"link"`
Description string `xml:"description"` Description string `xml:"description"`
PubDate string `xml:"pubDate"` PubDate string `xml:"pubDate"`
} }
func urlToFeed(url string) (RSSFeed, error) { func urlToFeed(url string) (RSSFeed, error) {
httpClient := http.Client{ Timeout: 10 * time.Second } httpClient := http.Client{Timeout: 10 * time.Second}
resp, err := httpClient.Get(url) resp, err := httpClient.Get(url)
if err != nil { if err != nil {

View File

@ -2,7 +2,9 @@ package main
import ( import (
"context" "context"
"database/sql"
"log" "log"
"strings"
"sync" "sync"
"time" "time"
@ -36,7 +38,9 @@ func startScraping(
func scrapeFeed(wg *sync.WaitGroup, db *database.Queries, feed database.Feed) { func scrapeFeed(wg *sync.WaitGroup, db *database.Queries, feed database.Feed) {
defer wg.Done() defer wg.Done()
_, err := db.MarkFeedAsFetched(context.Background(), feed.ID) ctx := context.Background()
_, err := db.MarkFeedAsFetched(ctx, feed.ID)
if err != nil { if err != nil {
log.Println("Error makrking feed as fetched:", err) log.Println("Error makrking feed as fetched:", err)
return return
@ -49,7 +53,34 @@ func scrapeFeed(wg *sync.WaitGroup, db *database.Queries, feed database.Feed) {
} }
for _, item := range rssFeed.Channel.Items { for _, item := range rssFeed.Channel.Items {
log.Println("Found post:", item.Title, "on feed", feed.Name) description := sql.NullString{}
if item.Description != "" {
description.String = item.Description
description.Valid = true
}
publishedAt, err := time.Parse(time.RFC1123Z, item.PubDate)
if err != nil {
log.Printf("Couldn't parse date %v with err %v\n", item.PubDate, err)
continue
}
now := time.Now().UTC()
_, err = db.CreatePost(ctx, database.CreatePostParams{
UpdatedAt: now,
CreatedAt: now,
Title: item.Title,
Description: description,
PublishedAt: publishedAt,
Url: item.Link,
FeedID: feed.ID,
})
if err != nil {
if strings.Contains(err.Error(), "UNIQUE constraint failed: posts.url") {
continue
}
log.Println("Failed to add post:", err)
}
} }
log.Printf("Feed %s collected, %v posts found", feed.Name, len(rssFeed.Channel.Items)) log.Printf("Feed %s collected, %v posts found", feed.Name, len(rssFeed.Channel.Items))
} }

21
sql/queries/posts.sql Normal file
View File

@ -0,0 +1,21 @@
-- name: CreatePost :one
INSERT INTO posts (
created_at,
updated_at,
title,
description,
published_at,
url,
feed_id)
VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING *;
-- name: GetPosts :many
SELECT * FROM posts;
-- name: GetPostsForUser :many
SELECT posts.* FROM posts
JOIN feed_follows ON posts.feed_id = feed_follows.id
WHERE feed_follows.user_id = ?
ORDER BY posts.published_at DESC
LIMIT ?;

14
sql/schema/006_posts.sql Normal file
View File

@ -0,0 +1,14 @@
-- +goose Up
CREATE TABLE posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_at DATE NOT NULL,
updated_at DATE NOT NULL,
title TEXT NOT NULL,
description TEXT,
published_at DATE NOT NULL,
url TEXT NOT NULL UNIQUE,
feed_id INTEGER NOT NULL REFERENCES feeds(id) ON DELETE CASCADE
);
-- +goose Down
DROP TABLE posts;