performance - Go-Couchbase: Bulk load utility speed issue -
i have created go utility populates data couchbase sql server.
the code follows:
package main import ( "database/sql" "log" "fmt" _ "github.com/denisenkom/go-mssqldb" "gopkg.in/couchbase/gocb.v1" ) func main() { var ( items1 []gocb.bulkop items2 []gocb.bulkop items3 []gocb.bulkop ) cluster, _ := gocb.connect("couchbase://localhost") bucket, _ := cluster.openbucket("example", "") condb, _ := sql.open("mssql", "server=.\\sqlexpress;port=62587; user id=sa;password=*******;") := 0; < 20; i++ { items1 = nil items2 = nil items3 = nil //get data sql server getdata(condb, &items1, &items2, &items3) // bulk load json couchbase data bucket insertdata(bucket, &items1) insertdata(bucket, &items2) insertdata(bucket, &items3) } err := bucket.close() if err != nil { fmt.println("error closing couchbase connection:", err) } err = condb.close() if err != nil { fmt.println("error closing mssql connection:", err) } } func getdata(condb *sql.db, items1 *[]gocb.bulkop, items2 *[]gocb.bulkop, items3 *[]gocb.bulkop) () { var id string var jsondata string var batch int query := "exec personlocation.dbo.uspgetpersonlocation" rows, err := condb.query(query) if err != nil { log.fatal(err) err = nil } rows.next() { err := rows.scan(&id,&jsondata,&batch) if err != nil { fmt.println("error:",err) } switch batch { case 1 : *items1 = append(*items1, &gocb.upsertop{key: id, value: jsondata}) case 2 : *items2 = append(*items2, &gocb.upsertop{key: id, value: jsondata}) case 3 : *items3 = append(*items3, &gocb.upsertop{key: id, value: jsondata}) } } } func insertdata(bucket *gocb.bucket, item *[]gocb.bulkop) (){ err := bucket.do(*item) if err != nil { fmt.println("error performing bulk insert:", err) } } in 1 iteration of loop, approximately 5100 records being inserted. ideally, should give speed of approx. 5000 inserts per second , if launch multiple instances of same application, insert speed should multiply , increase.
initially, worked fine (i launched 6 instances @ time , insert speed approx.15000 inserts per second) after month or so, became unstable , insert speed has decreased drastically. there hasn’t been change in hardware configuration well. not able find out might causing become slow. please help..
Comments
Post a Comment