performance - Go-Couchbase: Bulk load utility speed issue -


i have created go utility populates data couchbase sql server.

the code follows:

package main import (       "database/sql"       "log"       "fmt"       _ "github.com/denisenkom/go-mssqldb"       "gopkg.in/couchbase/gocb.v1"       )  func main() { var (     items1 []gocb.bulkop      items2 []gocb.bulkop      items3 []gocb.bulkop         )  cluster, _ := gocb.connect("couchbase://localhost") bucket, _ := cluster.openbucket("example", "") condb, _ := sql.open("mssql", "server=.\\sqlexpress;port=62587; user id=sa;password=*******;")  := 0; < 20; i++ {     items1 = nil     items2 = nil     items3 = nil     //get data sql server     getdata(condb, &items1, &items2, &items3)     // bulk load json couchbase data bucket      insertdata(bucket, &items1)     insertdata(bucket, &items2)     insertdata(bucket, &items3) }  err := bucket.close() if err != nil {    fmt.println("error closing couchbase connection:", err)    }  err = condb.close() if err != nil {    fmt.println("error closing mssql connection:", err)    } }  func getdata(condb *sql.db, items1 *[]gocb.bulkop, items2 *[]gocb.bulkop, items3 *[]gocb.bulkop) () {     var id string     var jsondata string     var batch int      query := "exec personlocation.dbo.uspgetpersonlocation"     rows, err := condb.query(query)     if err != nil {        log.fatal(err)        err = nil        }     rows.next() {      err := rows.scan(&id,&jsondata,&batch)      if err != nil {          fmt.println("error:",err)      }      switch batch {          case 1 :           *items1 = append(*items1, &gocb.upsertop{key: id, value: jsondata})          case 2 :          *items2 = append(*items2, &gocb.upsertop{key: id, value: jsondata})          case 3 :          *items3 = append(*items3, &gocb.upsertop{key: id, value: jsondata})      }     }    }  func insertdata(bucket *gocb.bucket, item *[]gocb.bulkop) (){     err := bucket.do(*item)     if err != nil {        fmt.println("error performing bulk insert:", err)        } } 

in 1 iteration of loop, approximately 5100 records being inserted. ideally, should give speed of approx. 5000 inserts per second , if launch multiple instances of same application, insert speed should multiply , increase.

initially, worked fine (i launched 6 instances @ time , insert speed approx.15000 inserts per second) after month or so, became unstable , insert speed has decreased drastically. there hasn’t been change in hardware configuration well. not able find out might causing become slow. please help..


Comments