gh-109845: Make test_ftplib more stable under load (GH-109912)

recv() can return partial data cut in the middle of a multibyte
character. Test raw binary data instead of data incorrectly decoded by parts.

Author: serhiy-storchaka

Lib/test/test_ftplib.py

@@ -32,7 +32,7 @@
 DEFAULT_ENCODING = 'utf-8'
 # the dummy data returned by server over the data channel when
 # RETR, LIST, NLST, MLSD commands are issued
-RETR_DATA = 'abcde12345\r\n' * 1000 + 'non-ascii char \xAE\r\n'
+RETR_DATA = 'abcde\xB9\xB2\xB3\xA4\xA6\r\n' * 1000
 LIST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n'
 NLST_DATA = 'foo\r\nbar\r\n non-ascii char \xAE\r\n'
 MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n"
@@ -67,11 +67,11 @@ class DummyDTPHandler(asynchat.async_chat):
     def __init__(self, conn, baseclass):
         asynchat.async_chat.__init__(self, conn)
         self.baseclass = baseclass
-        self.baseclass.last_received_data = ''
+        self.baseclass.last_received_data = bytearray()
         self.encoding = baseclass.encoding
 
     def handle_read(self):
-        new_data = self.recv(1024).decode(self.encoding, 'replace')
+        new_data = self.recv(1024)
         self.baseclass.last_received_data += new_data
 
     def handle_close(self):
@@ -107,7 +107,7 @@ def __init__(self, conn, encoding=DEFAULT_ENCODING):
         self.in_buffer = []
         self.dtp = None
         self.last_received_cmd = None
-        self.last_received_data = ''
+        self.last_received_data = bytearray()
         self.next_response = ''
         self.next_data = None
         self.rest = None
@@ -590,19 +590,17 @@ def test_abort(self):
         self.client.abort()
 
     def test_retrbinary(self):
-        def callback(data):
-            received.append(data.decode(self.client.encoding))
         received = []
-        self.client.retrbinary('retr', callback)
-        self.check_data(''.join(received), RETR_DATA)
+        self.client.retrbinary('retr', received.append)
+        self.check_data(b''.join(received),
+                        RETR_DATA.encode(self.client.encoding))
 
     def test_retrbinary_rest(self):
-        def callback(data):
-            received.append(data.decode(self.client.encoding))
         for rest in (0, 10, 20):
             received = []
-            self.client.retrbinary('retr', callback, rest=rest)
-            self.check_data(''.join(received), RETR_DATA[rest:])
+            self.client.retrbinary('retr', received.append, rest=rest)
+            self.check_data(b''.join(received),
+                            RETR_DATA[rest:].encode(self.client.encoding))
 
     def test_retrlines(self):
         received = []
@@ -612,7 +610,8 @@ def test_retrlines(self):
     def test_storbinary(self):
         f = io.BytesIO(RETR_DATA.encode(self.client.encoding))
         self.client.storbinary('stor', f)
-        self.check_data(self.server.handler_instance.last_received_data, RETR_DATA)
+        self.check_data(self.server.handler_instance.last_received_data,
+                        RETR_DATA.encode(self.server.encoding))
         # test new callback arg
         flag = []
         f.seek(0)
@@ -631,7 +630,8 @@ def test_storlines(self):
         data = RETR_DATA.replace('\r\n', '\n').encode(self.client.encoding)
         f = io.BytesIO(data)
         self.client.storlines('stor', f)
-        self.check_data(self.server.handler_instance.last_received_data, RETR_DATA)
+        self.check_data(self.server.handler_instance.last_received_data,
+                        RETR_DATA.encode(self.server.encoding))
         # test new callback arg
         flag = []
         f.seek(0)
@@ -649,7 +649,7 @@ def test_nlst(self):
 
     def test_dir(self):
         l = []
-        self.client.dir(lambda x: l.append(x))
+        self.client.dir(l.append)
         self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', ''))
 
     def test_mlsd(self):
@@ -889,12 +889,10 @@ def test_makepasv(self):
 
     def test_transfer(self):
         def retr():
-            def callback(data):
-                received.append(data.decode(self.client.encoding))
             received = []
-            self.client.retrbinary('retr', callback)
-            self.assertEqual(len(''.join(received)), len(RETR_DATA))
-            self.assertEqual(''.join(received), RETR_DATA)
+            self.client.retrbinary('retr', received.append)
+            self.assertEqual(b''.join(received),
+                             RETR_DATA.encode(self.client.encoding))
         self.client.set_pasv(True)
         retr()
         self.client.set_pasv(False)