Stop recording all mapper meta (mars-project#2900)

Author: Xuye (Chris) Qin

.codacy.yml

@@ -8,7 +8,7 @@ exclude_paths:
   - 'bin'
   - 'docs'
   - 'mars/_version.py'
-  - 'mars/lib'
+  - 'mars/lib/**'
   - 'mars/web/static'
   - 'misc'
   - 'versioneer.py'

.codecov.yml

@@ -28,6 +28,7 @@ ignore:
   - "mars/lib/enum.py"
   - "mars/lib/six.py"
   - "mars/lib/nvutils.py"
+  - "mars/lib/ordered_set.py"
   - "mars/lib/tblib"
   - "mars/lib/uhashring"
   - "mars/lib/version.py"

LICENSE

@@ -259,6 +259,8 @@ MIT License
 - sqlalchemy:1.4.0
 - async-lru:1.0.2
 - setuptools:55.0.0
+- bloom-filter2:2.0
+- ordered-set:4.1.0
 - react:17.0.0
 - react-dom:17.0.0
 - react-router-dom:5.2.0
@@ -267,7 +269,6 @@ MIT License
 - dagre-d3:0.6.4
 - cytoscape:3.19.1
 - cytoscape-dagre:2.3.2
-- bloom-filter2: 2.0
 
 
 ISC License

asv_bench/benchmarks/graph_assigner.py

@@ -33,7 +33,9 @@ def setup(self):
         df2 = md.DataFrame(
             mt.random.rand(num_rows, 4, chunk_size=10), columns=list("abcd")
         )
-        merged_df = df1.merge(df2, left_on="a", right_on="a")
+        merged_df = df1.merge(
+            df2, left_on="a", right_on="a", auto_merge="none", bloom_filter=False
+        )
         graph = TileableGraph([merged_df.data])
         next(TileableGraphBuilder(graph).build())
         self.chunk_graph = next(ChunkGraphBuilder(graph, fuse_enabled=False).build())

asv_bench/benchmarks/learn/model_selection.py

@@ -0,0 +1,33 @@
+# Copyright 1999-2022 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mars
+import mars.tensor as mt
+from mars.learn.model_selection import train_test_split
+
+
+class ModelSelectionSuite:
+    """
+    Benchmark learn model selection.
+    """
+
+    def setup(self):
+        self._session = mars.new_session()
+
+    def teardown(self):
+        self._session.stop_server()
+
+    def time_train_test_split(self):
+        t = mt.random.rand(10_000, 10, chunk_size=200)
+        train_test_split(t, test_size=0.3, session=self._session)

mars/dataframe/utils.py

@@ -1363,16 +1363,18 @@ def _concat_chunks(merge_chunks: List[ChunkType], output_index: int):
     n_split = []
     out_chunks = []
     for chunk, chunk_memory_size in zip(df_or_series.chunks, memory_sizes):
-        if acc_memory_size + chunk_memory_size > to_merge_size:
-            if len(to_merge_chunks) > 0:
-                # adding current chunk would exceed the maximum,
-                # concat previous chunks
-                merged_chunk = _concat_chunks(to_merge_chunks, len(n_split))
-                out_chunks.append(merged_chunk)
-                n_split.append(merged_chunk.shape[0])
-                # reset
-                acc_memory_size = 0
-                to_merge_chunks = []
+        if (
+            acc_memory_size + chunk_memory_size > to_merge_size
+            and len(to_merge_chunks) > 0
+        ):
+            # adding current chunk would exceed the maximum,
+            # concat previous chunks
+            merged_chunk = _concat_chunks(to_merge_chunks, len(n_split))
+            out_chunks.append(merged_chunk)
+            n_split.append(merged_chunk.shape[0])
+            # reset
+            acc_memory_size = 0
+            to_merge_chunks = []
 
         to_merge_chunks.append(chunk)
         acc_memory_size += chunk_memory_size