feat: add basic code
This commit is contained in:
commit
96368b78f6
SSH Key Fingerprint:
SHA256:g/OyGvi2pcd8ub9mqge/ohmDP0fZX/xOPWPIcM+9XpI
24
go.mod
Normal file
24
go.mod
Normal file
@ -0,0 +1,24 @@
|
||||
module neptune_project
|
||||
|
||||
go 1.22.0
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/andybalholm/brotli v1.1.1 // indirect
|
||||
github.com/brianvoe/gofakeit/v7 v7.0.4 // indirect
|
||||
github.com/go-sql-driver/mysql v1.8.1 // indirect
|
||||
github.com/gofiber/fiber/v2 v2.52.5 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mroth/weightedrand v1.0.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/segmentio/kafka-go v0.4.47 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.56.0 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
golang.org/x/sys v0.26.0 // indirect
|
||||
)
|
||||
96
go.sum
Normal file
96
go.sum
Normal file
@ -0,0 +1,96 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/brianvoe/gofakeit/v7 v7.0.4 h1:Mkxwz9jYg8Ad8NvT9HA27pCMZGFQo08MK6jD0QTKEww=
|
||||
github.com/brianvoe/gofakeit/v7 v7.0.4/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
|
||||
github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mroth/weightedrand v1.0.0 h1:V8JeHChvl2MP1sAoXq4brElOcza+jxLkRuwvtQu8L3E=
|
||||
github.com/mroth/weightedrand v1.0.0/go.mod h1:3p2SIcC8al1YMzGhAIoXD+r9olo/g/cdJgAD905gyNE=
|
||||
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0=
|
||||
github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.56.0 h1:bEZdJev/6LCBlpdORfrLu/WOZXXxvrUQSiyniuaoW8U=
|
||||
github.com/valyala/fasthttp v1.56.0/go.mod h1:sReBt3XZVnudxuLOx4J/fMrJVorWRiWY2koQKgABiVI=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
446
main.go
Normal file
446
main.go
Normal file
@ -0,0 +1,446 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/brianvoe/gofakeit/v7"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/mroth/weightedrand"
|
||||
"github.com/segmentio/kafka-go"
|
||||
)
|
||||
|
||||
func ByteCountSI(b int) string {
|
||||
const unit = 1000
|
||||
if b < unit {
|
||||
return fmt.Sprintf("%dB", b)
|
||||
}
|
||||
div, exp := int(unit), 0
|
||||
for n := b / unit; n >= unit; n /= unit {
|
||||
div *= unit
|
||||
exp++
|
||||
}
|
||||
return fmt.Sprintf("%.1f%cB",
|
||||
float64(b)/float64(div), "kMGTPE"[exp])
|
||||
}
|
||||
|
||||
func main() {
|
||||
log.Println("Neptune Project v0.1")
|
||||
|
||||
DATABASE_HOST, ok := os.LookupEnv("DATABASE_HOST")
|
||||
if !ok {
|
||||
log.Fatalln("DATABASE_HOST_NOT_FOUND")
|
||||
}
|
||||
|
||||
DATABASE_PORT, ok := os.LookupEnv("DATABASE_PORT")
|
||||
if !ok {
|
||||
log.Fatalln("DATABASE_PORT_NOT_FOUND")
|
||||
}
|
||||
|
||||
DATABASE_USER, ok := os.LookupEnv("DATABASE_USER")
|
||||
if !ok {
|
||||
log.Fatalln("DATABASE_USER_NOT_FOUND")
|
||||
}
|
||||
|
||||
DATABASE_PASS, ok := os.LookupEnv("DATABASE_PASS")
|
||||
if !ok {
|
||||
log.Fatalln("DATABASE_PASS_NOT_FOUND")
|
||||
}
|
||||
|
||||
DATABASE_SCHEMA, ok := os.LookupEnv("DATABASE_SCHEMA")
|
||||
if !ok {
|
||||
log.Fatalln("DATABASE_SCHEMA_NOT_FOUND")
|
||||
}
|
||||
|
||||
cfg := mysql.Config{
|
||||
User: DATABASE_USER,
|
||||
Passwd: DATABASE_PASS,
|
||||
Net: "tcp",
|
||||
Addr: DATABASE_HOST + ":" + DATABASE_PORT,
|
||||
DBName: DATABASE_SCHEMA,
|
||||
AllowNativePasswords: true,
|
||||
}
|
||||
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
log.Fatalln("DATABASE_OPEN_FAILED")
|
||||
}
|
||||
|
||||
err = db.Ping()
|
||||
if err != nil {
|
||||
log.Fatalln("DATABASE_CONNECTION_FAILED")
|
||||
}
|
||||
|
||||
stopConn := false
|
||||
log.Println("Creating large DB connection pool... Please wait...")
|
||||
for i := 0; i < 7000; i++ {
|
||||
db, err := sql.Open("mysql", cfg.FormatDSN())
|
||||
if err != nil {
|
||||
log.Println("DATABASE_OPEN_FAILED")
|
||||
}
|
||||
|
||||
_, err = db.Query("SELECT 1")
|
||||
if err != nil {
|
||||
stopConn = true
|
||||
}
|
||||
}
|
||||
|
||||
if stopConn {
|
||||
log.Println("DATABASE_CONNECTION_FAILED")
|
||||
log.Println("Ctrl+C to terminate DB connection pool")
|
||||
<-make(chan int)
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
DisableStartupMessage: true,
|
||||
})
|
||||
|
||||
go func() {
|
||||
app.Post("/api/streams/persist", func(c *fiber.Ctx) error {
|
||||
data := struct {
|
||||
GeneratedUserId string `json:"generated_user_id"`
|
||||
}{}
|
||||
|
||||
data.GeneratedUserId = gofakeit.UUID()
|
||||
|
||||
stmt, err := db.Prepare("INSERT INTO user (user_id, preferred_category, created_at) VALUES (?, ?, ?)")
|
||||
if err != nil {
|
||||
log.Println("DATABASE_PREPARE_FAILED")
|
||||
}
|
||||
|
||||
stmt.Exec(data.GeneratedUserId, "Test Category", time.Now())
|
||||
stmt.Close()
|
||||
|
||||
for j := 0; j < 10; j++ {
|
||||
stmt2, err := db.Prepare("INSERT INTO purchase (user_id, purchase_id, item_id, item_category, created_at) VALUES (?, ?, ?, ?, ?)")
|
||||
|
||||
if err != nil {
|
||||
log.Println("DATABASE_PREPARE_FAILED")
|
||||
}
|
||||
|
||||
stmt2.Exec(data.GeneratedUserId, gofakeit.UUID(), gofakeit.UUID(), "Test Category", time.Now())
|
||||
stmt2.Close()
|
||||
}
|
||||
|
||||
return c.JSON(data)
|
||||
})
|
||||
|
||||
for {
|
||||
preferred_categories, _ := weightedrand.NewChooser(
|
||||
weightedrand.Choice{Item: "Development", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Novel", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Cookbook", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Journal", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Art", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "History", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Science", Weight: gofakeit.UintN(10) + 1},
|
||||
)
|
||||
|
||||
for i := 1; i <= 20*60; i++ {
|
||||
stmt, err := db.Prepare("INSERT INTO user (user_id, preferred_category, created_at) VALUES (?, ?, ?)")
|
||||
if err != nil {
|
||||
log.Println("DATABASE_PREPARE_FAILED")
|
||||
}
|
||||
|
||||
user_id := gofakeit.UUID()
|
||||
preferred_category := preferred_categories.Pick().(string)
|
||||
|
||||
stmt.Exec(user_id, preferred_category, time.Now())
|
||||
stmt.Close()
|
||||
|
||||
for j := 0; j < gofakeit.IntN(6); j++ {
|
||||
stmt2, err := db.Prepare("INSERT INTO purchase (user_id, purchase_id, item_id, item_category, created_at) VALUES (?, ?, ?, ?, ?)")
|
||||
|
||||
if err != nil {
|
||||
log.Println("DATABASE_PREPARE_FAILED")
|
||||
}
|
||||
|
||||
preferred_category2 := preferred_categories.Pick().(string)
|
||||
stmt2.Exec(user_id, gofakeit.UUID(), gofakeit.UUID(), preferred_category2, time.Now())
|
||||
stmt2.Close()
|
||||
}
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
KAFKA_BROKERS, ok := os.LookupEnv("KAFKA_BROKERS")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
KAFKA_TOPIC, ok := os.LookupEnv("KAFKA_TOPIC")
|
||||
if !ok {
|
||||
log.Fatalln("KAFKA_TOPIC_NOT_FOUND")
|
||||
}
|
||||
|
||||
writer := &kafka.Writer{
|
||||
Addr: kafka.TCP(strings.Split(KAFKA_BROKERS, ",")...),
|
||||
Topic: KAFKA_TOPIC,
|
||||
Balancer: &kafka.LeastBytes{},
|
||||
}
|
||||
|
||||
app.Post("/api/streams/telemetry", func(c *fiber.Ctx) error {
|
||||
data := struct {
|
||||
GeneratedUserId string `json:"generated_user_id"`
|
||||
GeneratedCategory string `json:"generated_category"`
|
||||
GeneratedSessionId string `json:"generated_session_id"`
|
||||
}{}
|
||||
|
||||
data.GeneratedUserId = gofakeit.UUID()
|
||||
data.GeneratedCategory = gofakeit.UUID()
|
||||
|
||||
messages := []kafka.Message{}
|
||||
for i := 0; i < 10; i++ {
|
||||
messages = append(messages, kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"login\",\"details\":{\"user_id\":\"%s\",\"user_name\":\"%s\",\"user_age\":%d},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
data.GeneratedUserId, gofakeit.Name(), gofakeit.IntRange(16, 65), "manual_generated", gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
})
|
||||
}
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
messages = append(messages,
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
"searchFilterSelector", data.GeneratedCategory, "HIDE_ME", gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
})
|
||||
}
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
messages = append(messages,
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
"categorySelector", data.GeneratedCategory, "HIDE_ME", gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
})
|
||||
}
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
messages = append(messages,
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":null,\"session_issuer\":null,\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
"categorySelector", data.GeneratedCategory, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
})
|
||||
}
|
||||
|
||||
data.GeneratedSessionId = gofakeit.UUID()
|
||||
click_events1 := []string{}
|
||||
click_events2 := []string{}
|
||||
|
||||
for j := 0; j < 5; j++ {
|
||||
click_events1 = append(click_events1, fmt.Sprintf("{\"type\":\"click\",\"details\":{\"element\":\".%s\",\"content\":{\"item_name\":\"%s\",\"item_category\":\"%s\",\"item_location\":\"%s\"}},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
"bookItem", gofakeit.Adjective()+" "+gofakeit.Noun(), data.GeneratedCategory, gofakeit.Street(), data.GeneratedSessionId, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00")))
|
||||
}
|
||||
|
||||
for j := 0; j < 5; j++ {
|
||||
click_events2 = append(click_events2, fmt.Sprintf("{\"type\":\"click\",\"details\":{\"element\":\".%s\",\"content\":{\"item_name\":\"%s\",\"item_category\":\"%s\",\"item_location\":\"%s\"}},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
"bookItem", gofakeit.Adjective()+" "+gofakeit.Noun(), data.GeneratedCategory, gofakeit.Street(), data.GeneratedSessionId, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00")))
|
||||
}
|
||||
|
||||
messages = append(messages,
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"login\",\"details\":{\"user_id\":\"%s\",\"user_name\":\"%s\",\"user_age\":%d},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.Username(), gofakeit.Name(), gofakeit.IntRange(16, 65), data.GeneratedSessionId, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
},
|
||||
kafka.Message{
|
||||
Value: []byte("[" + strings.Join(click_events1, ",") + "]"),
|
||||
},
|
||||
kafka.Message{
|
||||
Value: []byte("[" + strings.Join(click_events2, ",") + "]"),
|
||||
})
|
||||
|
||||
writer.WriteMessages(context.Background(),
|
||||
messages...,
|
||||
)
|
||||
|
||||
return c.JSON(data)
|
||||
})
|
||||
|
||||
for {
|
||||
is_anon := gofakeit.IntN(10) > 3
|
||||
preferred_categories, _ := weightedrand.NewChooser(
|
||||
weightedrand.Choice{Item: "Development", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Novel", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Cookbook", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Journal", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Art", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "History", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "Science", Weight: gofakeit.UintN(10) + 1},
|
||||
)
|
||||
|
||||
for i := 1; i <= 100*60*5; i++ {
|
||||
click_events := []string{}
|
||||
click_event_count := gofakeit.IntRange(1, 5)
|
||||
if is_anon {
|
||||
for j := 0; j < click_event_count; j++ {
|
||||
click_events = append(click_events, fmt.Sprintf("{\"type\":\"click\",\"details\":{\"element\":\".%s\",\"content\":{\"item_name\":\"%s\",\"item_category\":\"%s\",\"item_location\":\"%s\"}},\"credentials\":{\"session_id\":null,\"session_issuer\":null,\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.RandomString([]string{"bookItem", "audioItem"}), gofakeit.Adjective()+" "+gofakeit.Noun(), preferred_categories.Pick().(string), gofakeit.Street(), gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00")))
|
||||
}
|
||||
|
||||
writer.WriteMessages(context.Background(),
|
||||
kafka.Message{
|
||||
Value: []byte("[" + strings.Join(click_events, ",") + "]"),
|
||||
},
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":null,\"session_issuer\":null,\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.RandomString([]string{"categorySelector", "searchFilterSelector"}), preferred_categories.Pick(), gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
},
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"move\",\"details\":{\"page\":\"/%s/%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.Adjective(), gofakeit.Noun(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
},
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"error\",\"details\":{\"page\":\"/%s/%s\",\"message\":\"%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.Adjective(), gofakeit.Noun(), gofakeit.LoremIpsumSentence(100), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
},
|
||||
)
|
||||
} else {
|
||||
session_id := gofakeit.UUID()
|
||||
for j := 0; j < click_event_count; j++ {
|
||||
click_events = append(click_events, fmt.Sprintf("{\"type\":\"click\",\"details\":{\"element\":\".%s\",\"content\":{\"item_name\":\"%s\",\"item_category\":\"%s\",\"item_location\":\"%s\"}},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.RandomString([]string{"bookItem", "audioItem"}), gofakeit.Adjective()+" "+gofakeit.Noun(), preferred_categories.Pick().(string), gofakeit.Street(), session_id, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00")))
|
||||
}
|
||||
|
||||
writer.WriteMessages(context.Background(),
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"login\",\"details\":{\"user_id\":\"%s\",\"user_name\":\"%s\",\"user_age\":%d},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.Username(), gofakeit.Name(), gofakeit.IntRange(16, 65), session_id, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
},
|
||||
kafka.Message{
|
||||
Value: []byte("[" + strings.Join(click_events, ",") + "]"),
|
||||
},
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"select\",\"details\":{\"element\":\"#%s\",\"value\":\"%s\"},\"credentials\":{\"session_id\":\"ubks_%s\",\"session_issuer\":\"mainsvc__vNeptune_1\",\"sys_trace_id\":\"ubks__mainsvc__vNeptune_1_%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.RandomString([]string{"categorySelector", "searchFilterSelector"}), preferred_categories.Pick(), session_id, gofakeit.UUID(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
},
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"move\",\"details\":{\"page\":\"/%s/%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.Adjective(), gofakeit.Noun(), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
},
|
||||
kafka.Message{
|
||||
Value: []byte(fmt.Sprintf("{\"type\":\"error\",\"details\":{\"page\":\"/%s/%s\",\"message\":\"%s\"},\"event_at\":\"%s\"}",
|
||||
gofakeit.Adjective(), gofakeit.Noun(), gofakeit.LoremIpsumSentence(100), time.Now().Format("2006-01-02T15:04:05.999999-07:00"))),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
app.Post("/api/streams/log", func(c *fiber.Ctx) error {
|
||||
data := struct {
|
||||
GeneratedReferrer string `json:"generated_referrer"`
|
||||
GeneratedCount int `json:"generated_count"`
|
||||
SampledLog string `json:"sampled_log"`
|
||||
}{}
|
||||
|
||||
data.GeneratedCount = 100000
|
||||
data.GeneratedReferrer = gofakeit.Noun() + ".test.com"
|
||||
|
||||
for i := 0; i < data.GeneratedCount; i++ {
|
||||
method := gofakeit.HTTPMethod()
|
||||
referrer := gofakeit.RandomString([]string{"http://", "https://"}) + data.GeneratedReferrer + ":" + gofakeit.RandomString([]string{"80", "443", "8080", "8443", "8000", "5000"}) + "/" + url.PathEscape(gofakeit.Adjective()) + "/" + url.PathEscape(gofakeit.Noun())
|
||||
|
||||
if method == "GET" || method == "HEAD" || method == "DELETE" {
|
||||
fmt.Printf("%s - [%s] %s %s %d %s %s %s \"%s\"\n",
|
||||
gofakeit.IPv4Address(),
|
||||
time.Now().Format("2006-01-02T15:04:05.999999 -07:00"),
|
||||
method,
|
||||
"/api/qry/"+gofakeit.Noun()+"?_qtt="+gofakeit.UUID()+"&_st="+url.QueryEscape(gofakeit.StreetName()),
|
||||
gofakeit.HTTPStatusCode(),
|
||||
"100MB",
|
||||
"100s",
|
||||
referrer,
|
||||
gofakeit.UserAgent())
|
||||
} else {
|
||||
fmt.Printf("%s - [%s] %s %s %d %s %s %s %s \"%s\"\n",
|
||||
gofakeit.IPv4Address(),
|
||||
time.Now().Format("2006-01-02T15:04:05.999999 -07:00"),
|
||||
method,
|
||||
"/api/products?_qcw="+gofakeit.UUID()+"&_rf="+gofakeit.UUID(),
|
||||
gofakeit.HTTPStatusCode(),
|
||||
"100MB",
|
||||
"100MB",
|
||||
"100s",
|
||||
referrer,
|
||||
gofakeit.UserAgent())
|
||||
}
|
||||
}
|
||||
|
||||
data.SampledLog = fmt.Sprintf("%s - [%s] %s %s %d %s %s %s %s \"%s\"\n",
|
||||
gofakeit.IPv4Address(),
|
||||
time.Now().Format("2006-01-02T15:04:05.999999 -07:00"),
|
||||
"POST",
|
||||
"/test",
|
||||
201,
|
||||
"1MB",
|
||||
"1B",
|
||||
"1s",
|
||||
"referrer",
|
||||
"test/"+gofakeit.UUID())
|
||||
|
||||
fmt.Print(data.SampledLog)
|
||||
return c.JSON(data)
|
||||
})
|
||||
|
||||
for {
|
||||
rxSeed := gofakeit.IntN(50) + 1
|
||||
txSeed := gofakeit.IntN(100) + 1
|
||||
lateSeed := gofakeit.IntN(12) + 1
|
||||
|
||||
referrers, _ := weightedrand.NewChooser(
|
||||
weightedrand.Choice{Item: "search.example.com", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "ad.unicorncorp.net", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "wiki.spreadinghoax.org", Weight: gofakeit.UintN(10) + 1},
|
||||
weightedrand.Choice{Item: "books.unicorn.com", Weight: 15},
|
||||
)
|
||||
|
||||
for i := 1; i <= 20*60; i++ {
|
||||
method := gofakeit.HTTPMethod()
|
||||
duration, _ := time.ParseDuration(strconv.Itoa(gofakeit.IntN(100*lateSeed)) + "ms")
|
||||
referrer := gofakeit.RandomString([]string{"http://", "https://"}) + referrers.Pick().(string) + ":" + gofakeit.RandomString([]string{"80", "443", "8080", "8443", "8000", "5000"}) + "/" + url.PathEscape(gofakeit.Adjective()) + "/" + url.PathEscape(gofakeit.Noun())
|
||||
|
||||
if method == "GET" || method == "HEAD" || method == "DELETE" {
|
||||
fmt.Printf("%s - [%s] %s %s %d %s %s %s \"%s\"\n",
|
||||
gofakeit.IPv4Address(),
|
||||
time.Now().Format("2006-01-02T15:04:05.999999 -07:00"),
|
||||
method,
|
||||
"/api/qry/"+gofakeit.Noun()+"?_qtt="+gofakeit.UUID()+"&_st="+url.QueryEscape(gofakeit.StreetName()),
|
||||
gofakeit.HTTPStatusCode(),
|
||||
ByteCountSI(gofakeit.IntN(1000*10*txSeed)),
|
||||
duration,
|
||||
referrer,
|
||||
gofakeit.UserAgent())
|
||||
} else {
|
||||
fmt.Printf("%s - [%s] %s %s %d %s %s %s %s \"%s\"\n",
|
||||
gofakeit.IPv4Address(),
|
||||
time.Now().Format("2006-01-02T15:04:05.999999 -07:00"),
|
||||
method,
|
||||
"/api/products?_qcw="+gofakeit.UUID()+"&_rf="+gofakeit.UUID(),
|
||||
gofakeit.HTTPStatusCode(),
|
||||
ByteCountSI(gofakeit.IntN(100*rxSeed)),
|
||||
ByteCountSI(gofakeit.IntN(1000*txSeed)),
|
||||
duration,
|
||||
referrer,
|
||||
gofakeit.UserAgent())
|
||||
}
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
app.Listen(":8080")
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user