Library/Vit.Db.DbMng/BaseDbMng.cs

@@ -74,17 +74,26 @@ namespace Vit.Db.DbMng
             Logger.Info(msg);
         }
 
-          
+
 
         /// <summary>
         /// 批量导入表数据（可以通过先停用索引，在导入数据后再启用来提高效率）
         /// </summary>
         /// <param name="dr"></param>
         /// <param name="tableName"></param>
+        /// <param name="tableRowCount"></param>
         /// <returns></returns>
-        protected virtual int BulkImport(IDataReader dr, string tableName) 
+        protected virtual int BulkImport(IDataReader dr, string tableName,int tableRowCount) 
         {
-            return conn.BulkImport(dr, tableName, commandTimeout: 0);
+            int index = 0;
+            return conn.BulkImport(dr, tableName
+                , onProcess: (cur, sum) =>
+                {
+                    index++;
+                    var process = (((float)sum) / tableRowCount * 100).ToString("f2");
+                    Log($"           {index}.[{process}%] {sum }/{tableRowCount}");
+                }
+                , useTransaction: true, commandTimeout: 0);
         }
 
 
@@ -167,32 +176,35 @@ namespace Vit.Db.DbMng
                     {
                         tbIndex++;
 
+                        int tableRowCount = conn.ExecuteScalar<int>("select count(*) from " + conn.Quote(tableName), commandTimeout: commandTimeout);
+
+
                         Log("");
                         Log($" ----[{tbIndex}/{sumTableCount}]backup table " + tableName);
+                        Log($"         rowCount: " + tableRowCount);
 
-                        int rowCount=0;
+                        int rowCount =0;
 
                         bool ignoreTable = false;
+           
 
-               
-                        #region 若表数据条数为0则跳过                   
-                        try
-                        {
-                            ignoreTable = 0 == conn.ExecuteScalar<int>("select count(*) from " + conn.Quote(tableName), commandTimeout: commandTimeout);
-                        }
-                        catch (Exception ex)
-                        {
-                            Logger.Info(ex);
-                        }
-                        #endregion
+                        //若表数据条数为0则跳过                   
+                        ignoreTable = 0 == tableRowCount;
 
                         if (!ignoreTable)
                         {
                             using (IDataReader dr = conn.ExecuteReader("select * from " + Quote(tableName), commandTimeout: commandTimeout))
                             {
                                 connSqlite.Sqlite_CreateTable(dr, tableName);
-
-                                rowCount = connSqlite.Import(dr, tableName, useTransaction: true, commandTimeout: commandTimeout);
+                                int index = 0;
+                                rowCount = connSqlite.Import(dr, tableName
+                                    , onProcess: (cur,sum) =>
+                                    {
+                                        index++;
+                                        var process = (((float)sum) / tableRowCount * 100).ToString("f2");
+                                        Log($"           {index}.[{process}%] {sum }/{tableRowCount}");
+                                    }
+                                    , useTransaction: true, commandTimeout: commandTimeout);
                             }
                         }
                         sumRowCount += rowCount;

Library/Vit.Db.DbMng/Extendsions/IDbConnection_MsSqlExtensions.cs

@@ -69,8 +69,7 @@ if Exists(select top 1 * from sysObjects where Id=OBJECT_ID(N'sqler_temp_filebuf
                          cmd.Connection = conn;
                          cmd.CommandText = sql;
 
-                         if (DapperConfig.CommandTimeout.HasValue)
-                             cmd.CommandTimeout = DapperConfig.CommandTimeout.Value;
+                         cmd.CommandTimeout = DapperConfig.CommandTimeout ?? 0;
 
                          using (var dr = cmd.ExecuteReader(CommandBehavior.SequentialAccess))
                          {

Library/Vit.Db.DbMng/MsSql/MsSqlDbMng.cs

@@ -530,8 +530,9 @@ deallocate   cDblogin
         /// </summary>
         /// <param name="dr"></param>
         /// <param name="tableName"></param>
+        /// <param name="tableRowCount"></param>
         /// <returns></returns>
-        protected override int BulkImport(IDataReader dr, string tableName)
+        protected override int BulkImport(IDataReader dr, string tableName,int tableRowCount)
         {
 
             #region (x.1)获取启用的索引
@@ -548,14 +549,14 @@ and T.name=@tableName
             #endregion
 
             if(sqlList.Count==0)
-                return conn.BulkImport(dr, tableName, commandTimeout: commandTimeout);
+                return base.BulkImport(dr, tableName, tableRowCount);
 
             try
             {
                 var sql_diableIndex = string.Join(" DISABLE;  ",sqlList) + " DISABLE;  ";
                 conn.Execute(sql_diableIndex, commandTimeout: commandTimeout);
 
-                return conn.BulkImport(dr, tableName, commandTimeout: commandTimeout);
+                return base.BulkImport(dr, tableName, tableRowCount);
             }
             finally
             {

Library/Vit.Db.DbMng/MySql/MySqlDbMng.cs

@@ -5,6 +5,7 @@ using System.Data;
 using System.IO;
 using System.Linq;
 using System.Text;
+using Vit.Core.Module.Log;
 using Vit.Core.Util.Common;
 using Vit.Extensions;
 using SqlConnection = MySql.Data.MySqlClient.MySqlConnection;
@@ -472,7 +473,38 @@ ORDER BY TABLE_NAME ASC, INDEX_NAME ASC;";
 
         #endregion
 
-         
+
+
+        #region SqlerBackuper
+        /// <summary>
+        /// 批量导入表数据（可以通过先停用索引，在导入数据后再启用来提高效率）
+        /// </summary>
+        /// <param name="dr"></param>
+        /// <param name="tableName"></param>
+        /// <param name="tableRowCount"></param>
+        /// <returns></returns>
+        protected override int BulkImport(IDataReader dr, string tableName, int tableRowCount)
+        {  
+            try
+            {         
+                conn.Execute("ALTER TABLE "+conn.Quote(tableName) +" DISABLE KEYS;", commandTimeout: commandTimeout);
+
+                return base.BulkImport(dr, tableName, tableRowCount);
+            }
+            finally
+            {
+                try
+                {
+                    conn.Execute("ALTER TABLE " + conn.Quote(tableName) + " ENABLE KEYS;", commandTimeout: commandTimeout);
+                }
+                catch (Exception ex)
+                {
+                    Logger.Error(ex);
+                }
+            }
+        }
+
+        #endregion
 
 
         #region BackupSqler

Library/Vit.Db.DbMng/SqlerFile/SqlerBackuper.cs

@@ -19,7 +19,7 @@ namespace Vit.Db.DbMng.SqlerFile
         public string dirPath;
         public IDbConnection conn;
         public Action<string> Log;
-        public Func<IDataReader, string, int> BulkImport;
+        public Func<IDataReader, string, int,int> BulkImport;
 
         public Regex sqlSplit = null;
 
@@ -124,12 +124,13 @@ namespace Vit.Db.DbMng.SqlerFile
                                 Log("");
                                 Log($" ----[{tbIndex}/{sumTableCount}]import table " + tableName);
 
+                                int tableRowCount = connSqlite.ExecuteScalar<int>("select count(*) from " + connSqlite.Quote(tableName), commandTimeout: commandTimeout);
                                 int rowCount=0;
-                                if (0<connSqlite.ExecuteScalar<int>("select count(*) from " + connSqlite.Quote(tableName), commandTimeout: commandTimeout))
+                                if (0< tableRowCount)
                                 {
                                     using (var dr = connSqlite.ExecuteReader("select * from " + connSqlite.Quote(tableName), commandTimeout: commandTimeout))
                                     {
-                                        rowCount = BulkImport(dr, tableName);
+                                        rowCount = BulkImport(dr, tableName,tableRowCount);
                                         sumRowCount += rowCount;
                                     }
                                 }

Sqler/Module/Sqler/Logical/DbPort/DataOutput.cs

@@ -176,25 +176,17 @@ namespace Sqler.Module.Sqler.Logical.DbPort
                         SendMsg(EMsgType.Nomal, "           [x.x.5]write data");
                         try
                         {
-                            while (true)
-                            {
-
-                                int rowCount = conn.BulkImport(dr, tableName, commandTimeout: DbPortLogical.commandTimeout, maxRowCount: DbPortLogical.batchRowCount);
-
 
-
-                                importedRowCount += rowCount;
-                                importedSumRowCount += rowCount;
-
-                                WriteProcess(importedRowCount);
-
-                                if (rowCount < DbPortLogical.batchRowCount)
+                            conn.BulkImport(dr, tableName
+                                , batchRowCount: DbPortLogical.batchRowCount
+                                , onProcess: (rowCount, sumRowCount) =>
                                 {
-                                    break;
-                                }
-
-                            }
+                                    importedRowCount += rowCount;
+                                    importedSumRowCount += rowCount;
 
+                                    WriteProcess(importedRowCount);
+                                }
+                                , commandTimeout: DbPortLogical.commandTimeout);
                         }
                         catch (Exception ex)
                         {

Sqler/Module/Sqler/Logical/DbPort/DbPortLogical.cs

@@ -24,7 +24,7 @@ namespace Sqler.Module.Sqler.Logical.DbPort
 
         public static int? commandTimeout => Vit.Orm.Dapper.DapperConfig.CommandTimeout;
 
-        public static readonly int batchRowCount = Vit.Core.Util.ConfigurationManager.ConfigurationManager.Instance.GetByPath<int?>("Sqler.DbPort_batchRowCount") ?? 100000;
+        public static readonly int batchRowCount = Vit.Core.Util.ConfigurationManager.ConfigurationManager.Instance.GetByPath<int?>("Sqler.DbPort_batchRowCount") ?? 500000;
   
 
         #region GetSqlRunConfig
@@ -247,24 +247,18 @@ namespace Sqler.Module.Sqler.Logical.DbPort
 
                     //(x.x.2)write data  
                     SendMsg(EMsgType.Nomal, "           [x.x.2]write data ");
-                    int importedRowCount = 0;
-                    while (true)
-                    {
-                        int rowCount = connSqlite.Import(dr, tableName, maxRowCount: batchRowCount, batchRowCount: 0);
-
-
-                        importedRowCount += rowCount;
-                        importedSumRowCount += rowCount;
-
-                        WriteProcess(importedRowCount);
+ 
+                    
+                    return connSqlite.Import(dr, tableName
+                         , batchRowCount: batchRowCount, onProcess: (rowCount, sumRowCount) =>
+                         {              
+                             importedSumRowCount += rowCount;
 
-                        if (rowCount < batchRowCount)
-                        {
-                            break;
-                        }
-                    }
+                             WriteProcess(sumRowCount);
+                         }
+                        , useTransaction: true, commandTimeout: commandTimeout); 
 
-                    return importedRowCount;
+            
 
                 };
 
@@ -711,7 +705,7 @@ namespace Sqler.Module.Sqler.Logical.DbPort
                 SendMsg(EMsgType.Title, "");
                 SendMsg(EMsgType.Title, "");
                 SendMsg(EMsgType.Title, "   Import success");
-                SendMsg(EMsgType.Title, "   sum row count:" + output.importedSumRowCount);
+                SendMsg(EMsgType.Title, "    table count: " + tableNames?.Count + ",  row count:" + output.importedSumRowCount);
                 SendMsg(EMsgType.Nomal, $"   耗时:{span.Hours}小时{span.Minutes}分{span.Seconds}秒{span.Milliseconds}毫秒");
                 #endregion
 

Sqler/appsettings.json

@@ -45,8 +45,8 @@
     /* Data文件夹的路径， 优先从arg第一个参数获取，其次是这里，若不指定则默认为"Data" */
     "//DataPath": "Data",
 
-    /* 导入导出工具配置，数据分批条数。若不指定则默认为 100000 */
-    "DbPort_batchRowCount": 100000
+    /* 导入导出工具配置，数据分批条数。若不指定则默认为 500000 */
+    "DbPort_batchRowCount": 500000
   },