SowaLabs / NpgsqlBulkCopy

Npgsql Bulk Copy

NpgsqlBulkCopy

Npgsql Bulk Copy

I simple demostration:

Scripts:

-- Database: northwind

-- DROP DATABASE northwind;

CREATE DATABASE northwind WITH OWNER = postgres ENCODING = 'UTF8' TABLESPACE = pg_default LC_COLLATE = 'English_United States.1252' LC_CTYPE = 'English_United States.1252' CONNECTION LIMIT = -1;

-- Table: employees

-- DROP TABLE employees;

CREATE TABLE employees ( "EmployeeID" smallint NOT NULL, "LastName" character varying(20) NOT NULL, "FirstName" character varying(10) NOT NULL, "Title" character varying(30), "TitleOfCourtesy" character varying(25), "BirthDate" date, "HireDate" date, "Address" character varying(60), "City" character varying(15), "Region" character varying(15), "PostalCode" character varying(10), "Country" character varying(15), "HomePhone" character varying(24), "Extension" character varying(4), "Photo" bytea, "Notes" text, "ReportsTo" smallint, "PhotoPath" character varying(255), CONSTRAINT pk_employees PRIMARY KEY ("EmployeeID") ) WITH ( OIDS=FALSE ); ALTER TABLE employees OWNER TO postgres;

Config file:

BulkCopyConfig.xml

9 127.0.0.1 5432 northwind postgres web@1234

Source:

    static unsafe void Main(string[] args)
    {
        var model = BulkCopyFactory.GetModel();
        var columnData = String.Join(",", new[]
                {
                    "EmployeeID",
                    "LastName",
                    "FirstName",
                    "Title",
                    "BirthDate",
                    "Address"
                }.Select(s => String.Format("\"{0}\"", s)));
        var tableName = "\"public\".\"employees\"";
        var batchSize = 10000;
        var recordSize = 10000000;

        PQNativeApi.LoadDLLDirectory(model.MajorVersion);

        switch (model.MajorVersion)
        {
            case PgVersions.PG8x:
                {
                    PQ8xNativeApi.openLocaleConn(model.Connection.Database.AsPointer(), model.Connection.UserName.AsPointer(), model.Connection.Password.AsPointer());
                    PQ8xNativeApi.setColumns(columnData.AsPointer());
                    PQ8xNativeApi.setTableName(tableName.AsPointer());
                    PQ8xNativeApi.setBatchSize(batchSize);

                    for (var i = 0; i < recordSize; i++)
                    {
                        var row = String.Join(",", new object[]
                        {
                            i + 1,
                            "'Elyor'",
                            "'Laipov'",
                            "'Software Developer'",
                            "now()",
                            "'Uzbekistan,Bukhara,Shofirkan'"
                        });

                        PQ8xNativeApi.addRow(row.AsPointer());
                    }

                    PQ8xNativeApi.runBulkCopy();
                    PQ8xNativeApi.closeConn();
                    PQ8xNativeApi.cleanUp();

                }
                break;
            case PgVersions.PQ9x:
                {
                    PQ9xNativeApi.openLocaleConn(model.Connection.Database.AsPointer(), model.Connection.UserName.AsPointer(), model.Connection.Password.AsPointer());
                    PQ9xNativeApi.setColumns(columnData.AsPointer());
                    PQ9xNativeApi.setTableName(tableName.AsPointer());
                    PQ9xNativeApi.setBatchSize(batchSize);

                    for (var i = 0; i < recordSize; i++)
                    {
                        var row = String.Join(",", new object[]
                        {
                            i + 1,
                            "'Elyor'",
                            "'Laipov'",
                            "'Software Developer'",
                            "now()",
                            "'Uzbekistan,Bukhara,Shofirkan'"
                        });

                        PQ9xNativeApi.addRow(row.AsPointer());
                    }

                    PQ9xNativeApi.runBulkCopy();
                    PQ9xNativeApi.closeConn();
                    PQ9xNativeApi.cleanUp();

                }
                break;
        }


        Console.WriteLine("Bulk copy successfully");
        Console.ReadKey();
    }

Performance:

Batch size:10.000 , Rows -> 10.000.000 records to load time ~5.8 min Batch size: 100.000 ,Rows-> 10.000.000 records to load time ~5.9 min