commit 96368b78f60a4a7acc416097f06634db3951560e Author: Minhyeok Park Date: Tue Oct 22 19:36:29 2024 +0900 feat: add basic code diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ba89a29 --- /dev/null +++ b/go.mod @@ -0,0 +1,24 @@ +module neptune_project + +go 1.22.0 + +require ( + filippo.io/edwards25519 v1.1.0 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/brianvoe/gofakeit/v7 v7.0.4 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect + github.com/gofiber/fiber/v2 v2.52.5 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mroth/weightedrand v1.0.0 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/segmentio/kafka-go v0.4.47 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.56.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + golang.org/x/sys v0.26.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..eac03b4 --- /dev/null +++ b/go.sum @@ -0,0 +1,96 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/brianvoe/gofakeit/v7 v7.0.4 h1:Mkxwz9jYg8Ad8NvT9HA27pCMZGFQo08MK6jD0QTKEww= +github.com/brianvoe/gofakeit/v7 v7.0.4/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= +github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mroth/weightedrand v1.0.0 h1:V8JeHChvl2MP1sAoXq4brElOcza+jxLkRuwvtQu8L3E= +github.com/mroth/weightedrand v1.0.0/go.mod h1:3p2SIcC8al1YMzGhAIoXD+r9olo/g/cdJgAD905gyNE= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= +github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.56.0 h1:bEZdJev/6LCBlpdORfrLu/WOZXXxvrUQSiyniuaoW8U= +github.com/valyala/fasthttp v1.56.0/go.mod h1:sReBt3XZVnudxuLOx4J/fMrJVorWRiWY2koQKgABiVI= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..fe7c09c --- /dev/null +++ b/main.go @@ -0,0 +1,446 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "log" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/brianvoe/gofakeit/v7" + "github.com/go-sql-driver/mysql" + "github.com/gofiber/fiber/v2" + "github.com/mroth/weightedrand" + "github.com/segmentio/kafka-go" +) + +func ByteCountSI(b int) string { + const unit = 1000 + if b < unit { + return fmt.Sprintf("%dB", b) + } + div, exp := int(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f%cB", + float64(b)/float64(div), "kMGTPE"[exp]) +} + +func main() { + log.Println("Neptune Project v0.1") + + DATABASE_HOST, ok := os.LookupEnv("DATABASE_HOST") + if !ok { + log.Fatalln("DATABASE_HOST_NOT_FOUND") + } + + DATABASE_PORT, ok := os.LookupEnv("DATABASE_PORT") + if !ok { + log.Fatalln("DATABASE_PORT_NOT_FOUND") + } + + DATABASE_USER, ok := os.LookupEnv("DATABASE_USER") + if !ok { + log.Fatalln("DATABASE_USER_NOT_FOUND") + } + + DATABASE_PASS, ok := os.LookupEnv("DATABASE_PASS") + if !ok { + log.Fatalln("DATABASE_PASS_NOT_FOUND") + } + + DATABASE_SCHEMA, ok := os.LookupEnv("DATABASE_SCHEMA") + if !ok { + log.Fatalln("DATABASE_SCHEMA_NOT_FOUND") + } + + cfg := mysql.Config{ + User: DATABASE_USER, + Passwd: DATABASE_PASS, + Net: "tcp", + Addr: DATABASE_HOST + ":" + DATABASE_PORT, + DBName: DATABASE_SCHEMA, + AllowNativePasswords: true, + } + + db, err := sql.Open("mysql", cfg.FormatDSN()) + if err != nil { + log.Fatalln("DATABASE_OPEN_FAILED") + } + + err = db.Ping() + if err != nil { + log.Fatalln("DATABASE_CONNECTION_FAILED") + } + + stopConn := false + log.Println("Creating large DB connection pool... Please wait...") + for i := 0; i < 7000; i++ { + db, err := sql.Open("mysql", cfg.FormatDSN()) + if err != nil { + log.Println("DATABASE_OPEN_FAILED") + } + + _, err = db.Query("SELECT 1") + if err != nil { + stopConn = true + } + } + + if stopConn { + log.Println("DATABASE_CONNECTION_FAILED") + log.Println("Ctrl+C to terminate DB connection pool") + <-make(chan int) + } + + app := fiber.New(fiber.Config{ + DisableStartupMessage: true, + }) + + go func() { + app.Post("/api/streams/persist", func(c *fiber.Ctx) error { + data := struct { + GeneratedUserId string `json:"generated_user_id"` + }{} + + data.GeneratedUserId = gofakeit.UUID() + + stmt, err := db.Prepare("INSERT INTO user (user_id, preferred_category, created_at) VALUES (?, ?, ?)") + if err != nil { + log.Println("DATABASE_PREPARE_FAILED") + } + + stmt.Exec(data.GeneratedUserId, "Test Category", time.Now()) + stmt.Close() + + for j := 0; j < 10; j++ { + stmt2, err := db.Prepare("INSERT INTO purchase (user_id, purchase_id, item_id, item_category, created_at) VALUES (?, ?, ?, ?, ?)") + + if err != nil { + log.Println("DATABASE_PREPARE_FAILED") + } + + stmt2.Exec(data.GeneratedUserId, gofakeit.UUID(), gofakeit.UUID(), "Test Category", time.Now()) + stmt2.Close() + } + + return c.JSON(data) + }) + + for { + preferred_categories, _ := weightedrand.NewChooser( + weightedrand.Choice{Item: "Development", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Novel", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Cookbook", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Journal", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Art", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "History", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Science", Weight: gofakeit.UintN(10) + 1}, + ) + + for i := 1; i <= 20*60; i++ { + stmt, err := db.Prepare("INSERT INTO user (user_id, preferred_category, created_at) VALUES (?, ?, ?)") + if err != nil { + log.Println("DATABASE_PREPARE_FAILED") + } + + user_id := gofakeit.UUID() + preferred_category := preferred_categories.Pick().(string) + + stmt.Exec(user_id, preferred_category, time.Now()) + stmt.Close() + + for j := 0; j < gofakeit.IntN(6); j++ { + stmt2, err := db.Prepare("INSERT INTO purchase (user_id, purchase_id, item_id, item_category, created_at) VALUES (?, ?, ?, ?, ?)") + + if err != nil { + log.Println("DATABASE_PREPARE_FAILED") + } + + preferred_category2 := preferred_categories.Pick().(string) + stmt2.Exec(user_id, gofakeit.UUID(), gofakeit.UUID(), preferred_category2, time.Now()) + stmt2.Close() + } + + time.Sleep(50 * time.Millisecond) + } + } + }() + + go func() { + KAFKA_BROKERS, ok := os.LookupEnv("KAFKA_BROKERS") + if !ok { + return + } + + KAFKA_TOPIC, ok := os.LookupEnv("KAFKA_TOPIC") + if !ok { + log.Fatalln("KAFKA_TOPIC_NOT_FOUND") + } + + writer := &kafka.Writer{ + Addr: kafka.TCP(strings.Split(KAFKA_BROKERS, ",")...), + Topic: KAFKA_TOPIC, + Balancer: &kafka.LeastBytes{}, + } + + app.Post("/api/streams/telemetry", func(c *fiber.Ctx) error { + data := struct { + GeneratedUserId string `json:"generated_user_id"` + GeneratedCategory string `json:"generated_category"` + GeneratedSessionId string `json:"generated_session_id"` + }{} + + data.GeneratedUserId = gofakeit.UUID() + data.GeneratedCategory = gofakeit.UUID() + + messages := []kafka.Message{} + for i := 0; i < 10; i++ { + messages = append(messages, kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"login\",\"details\":{\"user_id\":\"%s\",\"user_name\":\"%s\",\"user_age\":%d},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + data.GeneratedUserId, gofakeit.Name(), gofakeit.IntRange(16, 65), "manual_generated", gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }) + } + + for i := 0; i < 5; i++ { + messages = append(messages, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + "searchFilterSelector", data.GeneratedCategory, "HIDE_ME", gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }) + } + + for i := 0; i < 5; i++ { + messages = append(messages, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + "categorySelector", data.GeneratedCategory, "HIDE_ME", gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }) + } + + for i := 0; i < 5; i++ { + messages = append(messages, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":null,\"session_issuer\":null,\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + "categorySelector", data.GeneratedCategory, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }) + } + + data.GeneratedSessionId = gofakeit.UUID() + click_events1 := []string{} + click_events2 := []string{} + + for j := 0; j < 5; j++ { + click_events1 = append(click_events1, fmt.Sprintf("{\"type\":\"click\",\"details\":{\"element\":\".%s\",\"content\":{\"item_name\":\"%s\",\"item_category\":\"%s\",\"item_location\":\"%s\"}},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + "bookItem", gofakeit.Adjective()+" "+gofakeit.Noun(), data.GeneratedCategory, gofakeit.Street(), data.GeneratedSessionId, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))) + } + + for j := 0; j < 5; j++ { + click_events2 = append(click_events2, fmt.Sprintf("{\"type\":\"click\",\"details\":{\"element\":\".%s\",\"content\":{\"item_name\":\"%s\",\"item_category\":\"%s\",\"item_location\":\"%s\"}},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + "bookItem", gofakeit.Adjective()+" "+gofakeit.Noun(), data.GeneratedCategory, gofakeit.Street(), data.GeneratedSessionId, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))) + } + + messages = append(messages, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"login\",\"details\":{\"user_id\":\"%s\",\"user_name\":\"%s\",\"user_age\":%d},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + gofakeit.Username(), gofakeit.Name(), gofakeit.IntRange(16, 65), data.GeneratedSessionId, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }, + kafka.Message{ + Value: []byte("[" + strings.Join(click_events1, ",") + "]"), + }, + kafka.Message{ + Value: []byte("[" + strings.Join(click_events2, ",") + "]"), + }) + + writer.WriteMessages(context.Background(), + messages..., + ) + + return c.JSON(data) + }) + + for { + is_anon := gofakeit.IntN(10) > 3 + preferred_categories, _ := weightedrand.NewChooser( + weightedrand.Choice{Item: "Development", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Novel", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Cookbook", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Journal", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Art", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "History", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "Science", Weight: gofakeit.UintN(10) + 1}, + ) + + for i := 1; i <= 100*60*5; i++ { + click_events := []string{} + click_event_count := gofakeit.IntRange(1, 5) + if is_anon { + for j := 0; j < click_event_count; j++ { + click_events = append(click_events, fmt.Sprintf("{\"type\":\"click\",\"details\":{\"element\":\".%s\",\"content\":{\"item_name\":\"%s\",\"item_category\":\"%s\",\"item_location\":\"%s\"}},\"credentials\":{\"session_id\":null,\"session_issuer\":null,\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + gofakeit.RandomString([]string{"bookItem", "audioItem"}), gofakeit.Adjective()+" "+gofakeit.Noun(), preferred_categories.Pick().(string), gofakeit.Street(), gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))) + } + + writer.WriteMessages(context.Background(), + kafka.Message{ + Value: []byte("[" + strings.Join(click_events, ",") + "]"), + }, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":null,\"session_issuer\":null,\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + gofakeit.RandomString([]string{"categorySelector", "searchFilterSelector"}), preferred_categories.Pick(), gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"move\",\"details\":{\"page\":\"/%s/%s\"},\"event_at\":\"%s\"}", + gofakeit.Adjective(), gofakeit.Noun(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"error\",\"details\":{\"page\":\"/%s/%s\",\"message\":\"%s\"},\"event_at\":\"%s\"}", + gofakeit.Adjective(), gofakeit.Noun(), gofakeit.LoremIpsumSentence(100), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }, + ) + } else { + session_id := gofakeit.UUID() + for j := 0; j < click_event_count; j++ { + click_events = append(click_events, fmt.Sprintf("{\"type\":\"click\",\"details\":{\"element\":\".%s\",\"content\":{\"item_name\":\"%s\",\"item_category\":\"%s\",\"item_location\":\"%s\"}},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + gofakeit.RandomString([]string{"bookItem", "audioItem"}), gofakeit.Adjective()+" "+gofakeit.Noun(), preferred_categories.Pick().(string), gofakeit.Street(), session_id, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))) + } + + writer.WriteMessages(context.Background(), + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"login\",\"details\":{\"user_id\":\"%s\",\"user_name\":\"%s\",\"user_age\":%d},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + gofakeit.Username(), gofakeit.Name(), gofakeit.IntRange(16, 65), session_id, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }, + kafka.Message{ + Value: []byte("[" + strings.Join(click_events, ",") + "]"), + }, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}", + gofakeit.RandomString([]string{"categorySelector", "searchFilterSelector"}), preferred_categories.Pick(), session_id, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"move\",\"details\":{\"page\":\"/%s/%s\"},\"event_at\":\"%s\"}", + gofakeit.Adjective(), gofakeit.Noun(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }, + kafka.Message{ + Value: []byte(fmt.Sprintf("{\"type\":\"error\",\"details\":{\"page\":\"/%s/%s\",\"message\":\"%s\"},\"event_at\":\"%s\"}", + gofakeit.Adjective(), gofakeit.Noun(), gofakeit.LoremIpsumSentence(100), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))), + }, + ) + } + + time.Sleep(10 * time.Millisecond) + } + } + }() + + go func() { + app.Post("/api/streams/log", func(c *fiber.Ctx) error { + data := struct { + GeneratedReferrer string `json:"generated_referrer"` + GeneratedCount int `json:"generated_count"` + SampledLog string `json:"sampled_log"` + }{} + + data.GeneratedCount = 100000 + data.GeneratedReferrer = gofakeit.Noun() + ".test.com" + + for i := 0; i < data.GeneratedCount; i++ { + method := gofakeit.HTTPMethod() + referrer := gofakeit.RandomString([]string{"http://", "https://"}) + data.GeneratedReferrer + ":" + gofakeit.RandomString([]string{"80", "443", "8080", "8443", "8000", "5000"}) + "/" + url.PathEscape(gofakeit.Adjective()) + "/" + url.PathEscape(gofakeit.Noun()) + + if method == "GET" || method == "HEAD" || method == "DELETE" { + fmt.Printf("%s - [%s] %s %s %d %s %s %s \"%s\"\n", + gofakeit.IPv4Address(), + time.Now().Format("2006-01-02T15:04:05.999999 -07:00"), + method, + "/api/qry/"+gofakeit.Noun()+"?_qtt="+gofakeit.UUID()+"&_st="+url.QueryEscape(gofakeit.StreetName()), + gofakeit.HTTPStatusCode(), + "100MB", + "100s", + referrer, + gofakeit.UserAgent()) + } else { + fmt.Printf("%s - [%s] %s %s %d %s %s %s %s \"%s\"\n", + gofakeit.IPv4Address(), + time.Now().Format("2006-01-02T15:04:05.999999 -07:00"), + method, + "/api/products?_qcw="+gofakeit.UUID()+"&_rf="+gofakeit.UUID(), + gofakeit.HTTPStatusCode(), + "100MB", + "100MB", + "100s", + referrer, + gofakeit.UserAgent()) + } + } + + data.SampledLog = fmt.Sprintf("%s - [%s] %s %s %d %s %s %s %s \"%s\"\n", + gofakeit.IPv4Address(), + time.Now().Format("2006-01-02T15:04:05.999999 -07:00"), + "POST", + "/test", + 201, + "1MB", + "1B", + "1s", + "referrer", + "test/"+gofakeit.UUID()) + + fmt.Print(data.SampledLog) + return c.JSON(data) + }) + + for { + rxSeed := gofakeit.IntN(50) + 1 + txSeed := gofakeit.IntN(100) + 1 + lateSeed := gofakeit.IntN(12) + 1 + + referrers, _ := weightedrand.NewChooser( + weightedrand.Choice{Item: "search.example.com", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "ad.unicorncorp.net", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "wiki.spreadinghoax.org", Weight: gofakeit.UintN(10) + 1}, + weightedrand.Choice{Item: "books.unicorn.com", Weight: 15}, + ) + + for i := 1; i <= 20*60; i++ { + method := gofakeit.HTTPMethod() + duration, _ := time.ParseDuration(strconv.Itoa(gofakeit.IntN(100*lateSeed)) + "ms") + referrer := gofakeit.RandomString([]string{"http://", "https://"}) + referrers.Pick().(string) + ":" + gofakeit.RandomString([]string{"80", "443", "8080", "8443", "8000", "5000"}) + "/" + url.PathEscape(gofakeit.Adjective()) + "/" + url.PathEscape(gofakeit.Noun()) + + if method == "GET" || method == "HEAD" || method == "DELETE" { + fmt.Printf("%s - [%s] %s %s %d %s %s %s \"%s\"\n", + gofakeit.IPv4Address(), + time.Now().Format("2006-01-02T15:04:05.999999 -07:00"), + method, + "/api/qry/"+gofakeit.Noun()+"?_qtt="+gofakeit.UUID()+"&_st="+url.QueryEscape(gofakeit.StreetName()), + gofakeit.HTTPStatusCode(), + ByteCountSI(gofakeit.IntN(1000*10*txSeed)), + duration, + referrer, + gofakeit.UserAgent()) + } else { + fmt.Printf("%s - [%s] %s %s %d %s %s %s %s \"%s\"\n", + gofakeit.IPv4Address(), + time.Now().Format("2006-01-02T15:04:05.999999 -07:00"), + method, + "/api/products?_qcw="+gofakeit.UUID()+"&_rf="+gofakeit.UUID(), + gofakeit.HTTPStatusCode(), + ByteCountSI(gofakeit.IntN(100*rxSeed)), + ByteCountSI(gofakeit.IntN(1000*txSeed)), + duration, + referrer, + gofakeit.UserAgent()) + } + + time.Sleep(50 * time.Millisecond) + } + } + }() + + app.Listen(":8080") +}