SIM800 FTPPUT image file fails 30% of the times, only part of the image arrives. (ESP32)

Hi, this sketch transfers an image.jpg file, preloaded in this case in a FAT partition of the ESP32 to my FTP server. It works more than 50% of the times. When it fails there are no error messages and the image arrives missing the bottom part.

The sketch is very big but the functions that do the file transfer are as bellow:

byte gprs_modem_function (String newPicName){   // newPicName = name of file at the FTP server
  char singlePixelBuffer;
  int transitPacketBufferSize = 1360;
  String transitPacketBuffer = "";
  word longDelay   = 18000;
  word mediumDelay = 12000;
  word shortDelay  =  5000;
  byte reply = 1;
  int i = 0;
  
  if(!FFat.begin(true)){
    pDBGln("Fat FS: Mount Failed");
    return false;
  }
  imageFile = FFat.open("/esp32pic.jpg");   //pic_name);   //name of the file preloaded in FAT partition
  
  while (i < 10 && reply == 1){ //Try 10 times...
    reply = sendATcommand("AT+CREG?","+CREG: 0,1","ERROR", shortDelay);
    i++;
    delay(1000);
  }
  if (reply == 0){
    reply = sendATcommand("AT+SAPBR=3,1,\"Contype\",\"GPRS\"","OK","ERROR", longDelay);
    if (reply == 0){
      reply = sendATcommand("AT+SAPBR=3,1,\"APN\",\"claro.claro.com\"", "OK", "ERROR", mediumDelay);
      if (reply == 0){
        //reply = sendATcommand("AT+SAPBR=3,1,\"USER\",\"claro\"", "OK", "ERROR", mediumDelay);
        if (reply == 0){
          //reply = sendATcommand("AT+SAPBR=3,1,\"PWD\",\"claro\"", "OK", "ERROR", mediumDelay);
          if (reply == 0){
            reply = 2;
            i = 0;
            while (i < 3 && reply == 2){ //Try 3 times...
              reply = sendATcommand("AT+SAPBR=1,1", "OK", "ERROR", longDelay);
              if (reply == 2){
                sendATcommand("AT+SAPBR=0,1", "OK", "ERROR", longDelay);
              }
              i++;
            }
            if (reply == 0){
              reply = sendATcommand("AT+SAPBR=2,1", "OK", "ERROR", shortDelay);
              if (reply == 0){
                reply = sendATcommand("AT+FTPCID=1", "OK", "ERROR", shortDelay);
                reply = sendATcommand("AT+CCLK?", "OK", "ERROR", shortDelay);
                if(newPicName==""){
                  picName = GSMTimeStamp + ".jpg";
                }else{
                  picName = newPicName;
                }
                if (reply == 0){
                  reply = sendATcommand("AT+FTPSERV=\"myftpserver.com\"", "OK", "ERROR", shortDelay);
                  if (reply == 0){
                    reply = sendATcommand("AT+FTPPORT=21", "OK", "ERROR", shortDelay);
                    if (reply == 0){
                      reply = sendATcommand("AT+FTPUN=\"station@myftpserver.com\"", "OK", "ERROR", shortDelay);
                      if (reply == 0){
                        reply = sendATcommand("AT+FTPPW=\"mypass\"", "OK", "ERROR", shortDelay);
                        if (reply == 0){
                          reply = sendATcommand("AT+FTPPUTNAME=\"" + String(picName) + "\"", "OK", "ERROR", shortDelay);                      
                          if (reply == 0){                                 
                            reply = sendATcommand("AT+FTPPUTPATH=\"/\"", "OK", "ERROR", shortDelay);
                            if (reply == 0){
                              unsigned int ptime = millis();
                              reply = sendATcommand("AT+FTPPUT=1", "+FTPPUT: 1,1", "+FTPPUT: 1,6", longDelay*4);
                              pDBGln("Time: " + String(millis() - ptime));
                              if (reply == 0){
                                if (imageFile) {
                                  int i = 0;
                                  word bytesTransfered = 0;                                  
                                  word packetCounter   = 0;
                                  word fileSize        = imageFile.size();
                                  word numberOfPackets = fileSize/transitPacketBufferSize;
                                  if((fileSize%transitPacketBufferSize)>0){
                                    numberOfPackets++;
                                  }                                  
                                  while(imageFile.available()>0){
                                    singlePixelBuffer = imageFile.read();
                                    transitPacketBuffer.concat(singlePixelBuffer);
                                    i++;
                                    bytesTransfered++;
                                    if (i == transitPacketBufferSize) {
                                      reply = sendATcommand("AT+FTPPUT=2," + String(transitPacketBufferSize) + "\r\n", "+FTPPUT:", "ERROR", longDelay);
                                      reply = sendATcommand(transitPacketBuffer + "\r\n", "OK", "ERROR", longDelay);
                                      transitPacketBuffer = "";
                                      i = 0;      
                                      packetCounter++; 
                                    }
                                  }
                                  if(transitPacketBuffer.length() != 0){
                                    reply = sendATcommand("AT+FTPPUT=2," + String(i) + "\r\n", "+FTPPUT:", "ERROR", longDelay);
                                    reply = sendATcommand(transitPacketBuffer + "\r\n", "OK", "ERROR", longDelay); 
                                    packetCounter++;   
                                  }
                                  reply = sendATcommand("AT+FTPPUT=2,0\r\n", "+FTPPUT: 1,", "ERROR", longDelay); 
                                  imageFile.close();                                   
                                  if(packetCounter!=numberOfPackets){
                                    reply = 2;
                                    pDBG("#----------"+String(picName)+"-->>Packet Count Error: ");
                                    pDBGln(numberOfPackets-packetCounter);                             
                                  }else{
                                    pDBG(String(picName));         
                                    pDBG(", SDsize: ");
                                    pDBG(picSize);
                                    pDBGln(" bytes");
                                  }
                                  if(bytesTransfered!=fileSize){
                                    pDBG("-------->>>Bytes Transfer Failure: ");
                                    pDBG(String(picName));
                                    pDBG(" Filesize: ");
                                    pDBG(fileSize);
                                    pDBG("  -Transfered: ");
                                    pDBGln(bytesTransfered);
                                  }
                                }
                              }
                            }
                          }
                        }
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }
  return reply;
}

byte sendATcommand(String ATcommand, String answer1, String answer2, unsigned int timeout){
  byte reply = 1;
  String content = "";
  char character;
 
  //Clean the modem input buffer
  while(SerialAT.available()>0){
    SerialAT.read();
  }
  //Send the atcommand to the modem
  SerialAT.println(ATcommand);
  delay(100);
  unsigned int timeprevious = millis();
  while((reply == 1) && ((millis() - timeprevious) < timeout)){
    while(SerialAT.available()>0) {
      character = SerialAT.read();
      content.concat(character);
      pDBG(character);
      delay(15);
    }

    //Stop reading conditions
    if (content.indexOf(answer1) != -1){
      reply = 0;
    }else if(content.indexOf(answer2) != -1){
      reply = 2;
    }else{
      //Nothing to do...
    }  
    if (content.indexOf("+FTPSIZE:") != -1){      //+FTPSIZE: 1,0,2324
      imgSizeFTP = content.substring(22).toInt();     
    }else if(content.indexOf("+CCLK:") != -1){
      GSMTimeStamp  = content.substring(10,12); 
      GSMTimeStamp += content.substring(13,15);
      GSMTimeStamp += content.substring(16,18);
      GSMTimeStamp += content.substring(19,21); 
      GSMTimeStamp += content.substring(22,24);
      GSMTimeStamp += content.substring(25,27);
      //pDBGln(GSMTimeStamp);
    }
  }
  pDBG(ATcommand);
  pDBG(" Resposta: ");
  pDBGln(content);
  return reply;
}

word chkFTPSize(String fileName){
  byte reply = 0;
  word longDelay   = 18000;
  word mediumDelay = 12000;
  word shortDelay  =  5000;
  int i;
  
  while (i < 10 && reply == 1){ //Try 10 times...
    reply = sendATcommand("AT+CREG?","+CREG: 0,1","ERROR", shortDelay);
    i++;
    delay(1000);
  }
  if (reply == 0){
    reply = sendATcommand("AT+SAPBR=3,1,\"Contype\",\"GPRS\"","OK","ERROR", longDelay);
    if (reply == 0){
      reply = sendATcommand("AT+SAPBR=3,1,\"APN\",\"claro.claro.com\"", "OK", "ERROR", mediumDelay);
      if (reply == 0){
        //reply = sendATcommand("AT+SAPBR=3,1,\"USER\",\"claro\"", "OK", "ERROR", mediumDelay);
        if (reply == 0){
          //reply = sendATcommand("AT+SAPBR=3,1,\"PWD\",\"claro\"", "OK", "ERROR", mediumDelay);
          if (reply == 0){
            reply = 2;
            i = 0;
            while (i < 3 && reply == 2){ //Try 3 times...
              reply = sendATcommand("AT+SAPBR=1,1", "OK", "ERROR", longDelay);
              if (reply == 2){
                sendATcommand("AT+SAPBR=0,1", "OK", "ERROR", longDelay);
              }
              i++;
            }
            if (reply == 0){
              reply = sendATcommand("AT+SAPBR=2,1", "OK", "ERROR", shortDelay);
              if (reply == 0){
                reply = sendATcommand("AT+FTPCID=1", "OK", "ERROR", shortDelay);
                if (reply == 0){
                  reply = sendATcommand("AT+FTPSERV=\"ftp.hidroflux.com\"", "OK", "ERROR", shortDelay);
                  if (reply == 0){
                    reply = sendATcommand("AT+FTPPORT=21", "OK", "ERROR", shortDelay);
                    if (reply == 0){
                      reply = sendATcommand("AT+FTPUN=\"station@iot.hidroflux.com\"", "OK", "ERROR", shortDelay);
                      if (reply == 0){
                        reply = sendATcommand("AT+FTPPW=\"Andromeda161\"", "OK", "ERROR", shortDelay);
                        if (reply == 0){
                          reply = sendATcommand("AT+FTPGETNAME=\""+fileName+"\"", "OK", "ERROR", shortDelay);
                          if (reply == 0){                  
                            reply = sendATcommand("AT+FTPGETPATH=\"/\"", "OK", "ERROR", shortDelay);                
                            if (reply == 0){
                              i=0;
                              do{reply = sendATcommand("AT+FTPSIZE", "+FTPSIZE: 1,", "ERROR", mediumDelay);
                                i++;
                              }while(imgSizeFTP==0 && reply!=2 && i<3);
                              return reply;
                            }
                          }
                        }
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}

Assistance welcome
Thanks
Paulo

2G modules can only push max chunks of 2K bytes reliably. If you are pushing more at a time, it will not arrive. How much you are pushing in each send?

Hi, thanks for the reply.

int transitPacketBufferSize = 1360;

But I tried 128, 256 and other buffer sizes but it is still not 100% reliable.

In a 6K image file for instance the chunk sizes would be as follow:
6000 / 1360 = 4 chunks of 1360 bytes and and 1 chunk of 560 bytes.

On tens and tens of tests I did in the example above chunks 1,2 and 3 always arrive ok.
So it is not data transfer error.

When it fails about 30% 4 do not arrive and 70% only chunk 5 fails to arrive.

In every upload that fails only 4 or 5 will fail and most only chunk 5 fails to arrive.
When 4 fails 5 fails as well although with no error messages.

When the image fails to fully arrive it will show a white band at the bottom, the size of the white band depends of how many chunks have failed to arrive.
In rare occasions chunk 3 also failed to arrive.

Curiously chunks that arrive are always intact, when a chunk fail it fails in its entirety.

I noticed that when chunks 1,2,3 and 4 are sent they are quickly acknowledged by the FTP server but chunk 5 always take a long time to be acknowledged and that is where it fails as if something had to tell the FTP server that no more data to send in spite of the following line:

reply = sendATcommand("AT+FTPPUT=2," + String(transitPacketBufferSize) + "\r\n", "+FTPPUT:", "ERROR", longDelay);
reply = sendATcommand(transitPacketBuffer + "\r\n", "OK", "ERROR", longDelay);

Please notice that in the last packet transitPacketBufferSize is updated appropriately so the FTP server is informed no more data in this chunk. After this last chunk the server seems like waiting for a longer time to acknowledge chunk was well received.

Then to close the file as no more data to transfer: AT+FTPPUT=2,0;

But when the image is fully transmitted successfully, it takes the same time for acknowledgement but somehow it succeeds.

By the way, as you can see on the sketch above I tried to implement FTPSIZE to check file size after upload so I could upload again if size do not match original file size but FTPSIZE fails too often sometimes forcing file to upload again in cases the file size was OK so the code is still there but I have abandoned the idea because the end result of successful file transfers is not much better.

Regards
Paulo

Hi Ravi, i did more research on it and maybe you can help on this one.
The code is almost the same as I provided above but it is the console debug that may provide interesting information.
I printed below the console printout for the last 2 chunks transferred.

Notice that I send the command: AT+FTPPUT=2,1360
And wait for “authorization”, as soon as I get a: +FTPPUT: 2,1360
That I understand to be an acknowledgement that I can then transfer 1360 bytes of data.
In the next line of code I send precisely 1360 bytes of data and get: OK in return.

On the last chunk on my test I only have 856 bytes to send to the FTP server.
So I first send the following command: AT+FTPPUT=2,856

What is quite strange is that on the console debug log, even before the command above is printed I get
a +FTPPUT: 2,856 what should only have come as a response from the command above signaling the FTP server is ready to receive the 856 bytes of data.

As you can see below after that I get the +FTPPUT: 2,856 and after that I send the 865 bytes of data.

But strangely after I send this last chunk of data I get: +FTPPUT: 1,1,1360
And I do not understand why, I was expecting to get a: +FTPPUT: 1,1,856 but it never arrives.

After that I get an OK and another +FTPPUT: 1,1,1360, another OK and then it takes almost 50 seconds and I finally get +FTPPUT: 1,64
Indicating timeout.
No wonder the image arrives often missing the last chunk.

See the relevant console output below:

AT+FTPPUT=2,1360 
+FTPPUT: 2,1360
OK
+FTPPUT: 1,1,1360
........1360 bytes binary data.................(manually removed for clarity)
OK

+FTPPUT: 1,1,1360
+FTPPUT: 2,856    <<---Should not be here!
AT+FTPPUT=2,856 
+FTPPUT: 2,856
OK
+FTPPUT: 1,1,1360    **<< ---WHAT IS THAT?** 
........856 bytes binary data..................(manually removed for clarity)
OK
+FTPPUT: 1,1,1360    **<< ---WHAT IS THAT?** Should be +FTPPUT: 1,1,856
OK
+FTPPUT: 1,64  <<-timeout, +FTPPUT: 1,1,856 fails to arrive

For convenience see function sketches below. I am now using an SD card as the source of the image file.

byte GSMPostPic() {
  String transitPacketBuffer = "";
  byte reply = 1;
  int  counter=1;
  int  i=0;  
  pDBGln("Starting...");
//  if(getPicture()){                         //ESP32CAM takes picture and stores in local FAT FS    
    reply = gprs_modem_function(picName);     //ESP32CAM transfers pic via Serial to ESPMaster
//  }  
  pDBG("The end. Transfer Status: ");
  pDBGln(reply);
  counter++;
  return reply;
}
                                   
byte gprs_modem_function (String newPicName){
  char singlePixelBuffer;
  int transitPacketBufferSize = 1360;
  String transitPacketBuffer = "";
  word longDelay   = 50000;
  word mediumDelay = 12000;
  word shortDelay  =  5000;
  byte reply = 1;
  int i = 0;
  
//  if(!FFat.begin(true)){
//    pDBGln("Fat FS: Mount Failed");
//    return false;
//  }

  //imageFile = FFat.open("/esp32pic.jpg");   //pic_name);
  File imageFile = SD.open("/testimage.jpg");   //pic_name);
  
  
  while (i < 10 && reply == 1){ //Try 10 times...
    reply = sendATcommand("AT+CREG?","+CREG: 0,1","ERROR", shortDelay);
    i++;
    delay(1000);
  }
  if (reply == 0){
    //reply = sendATcommand("AT+SAPBR=3,1,\"Contype\",\"GPRS\"","OK","ERROR", longDelay);
    if (reply == 0){
      //reply = sendATcommand("AT+SAPBR=3,1,\"APN\",\"claro.claro.com\"", "OK", "ERROR", mediumDelay);
      if (reply == 0){
        //reply = sendATcommand("AT+SAPBR=3,1,\"USER\",\"claro\"", "OK", "ERROR", mediumDelay);
        if (reply == 0){
          //reply = sendATcommand("AT+SAPBR=3,1,\"PWD\",\"claro\"", "OK", "ERROR", mediumDelay);
          if (reply == 0){
            reply = 2;
            i = 0;
            while (i < 3 && reply == 2){ //Try 3 times...
              reply = sendATcommand("AT+SAPBR=1,1", "OK", "ERROR", longDelay);
              if (reply == 2){
                sendATcommand("AT+SAPBR=0,1", "OK", "ERROR", longDelay);
              }
              i++;
            }
            if (reply == 0){
              reply = sendATcommand("AT+SAPBR=2,1", "OK", "ERROR", shortDelay);
              if (reply == 0){
                reply = sendATcommand("AT+FTPCID=1", "OK", "ERROR", shortDelay);

                reply = sendATcommand("AT+FTPMODE=1", "OK", "ERROR", shortDelay);
                reply = sendATcommand("AT+FTPTYPE=\"I\"", "OK", "ERROR", shortDelay);
                reply = sendATcommand("AT+FTPPUTOPT=\"STOR\"", "OK", "ERROR", shortDelay);

                picName = "image" + String(counter) + ".jpg";
                counter++;
                
                //if(newPicName==""){
                //  picName = GSMTimeStamp + ".jpg";
                //}else{
                //  picName = newPicName;
                //}
                
                if (reply == 0){
                  reply = sendATcommand("AT+FTPSERV=\"myftpserver.com\"", "OK", "ERROR", shortDelay);
                  if (reply == 0){
                    reply = sendATcommand("AT+FTPPORT=21", "OK", "ERROR", shortDelay);
                    if (reply == 0){
                      reply = sendATcommand("AT+FTPUN=\"user@myftpserver.com\"", "OK", "ERROR", shortDelay);
                      if (reply == 0){
                        reply = sendATcommand("AT+FTPPW=\"mypass\"", "OK", "ERROR", shortDelay);
                        if (reply == 0){
                          reply = sendATcommand("AT+FTPPUTNAME=\"" + String(picName) + "\"", "OK", "ERROR", shortDelay);                      
                          if (reply == 0){                                 
                            reply = sendATcommand("AT+FTPPUTPATH=\"/\"", "OK", "ERROR", shortDelay);
                            if (reply == 0){
                              unsigned int ptime = millis();
                              reply = sendATcommand("AT+FTPPUT=1", "+FTPPUT: 1,1", "+FTPPUT: 1,6", longDelay*4);
                              pDBGln("Time: " + String(millis() - ptime));
                              if (reply == 0){
                                if (imageFile) {
                                  int i = 0;
                                  word bytesTransfered = 0;                                  
                                  word packetCounter   = 0;
                                  word fileSize        = imageFile.size();
                                  word numberOfPackets = fileSize/transitPacketBufferSize;
                                  if((fileSize%transitPacketBufferSize)>0){
                                    numberOfPackets++;
                                  }                                  
                                  while(imageFile.available()>0){
                                    singlePixelBuffer = imageFile.read();
                                    transitPacketBuffer.concat(singlePixelBuffer);
                                    i++;
                                    bytesTransfered++;
                                    if (i == transitPacketBufferSize) {
                                      reply = sendATcommand("AT+FTPPUT=2," + String(transitPacketBufferSize), "+FTPPUT: 2," + String(transitPacketBufferSize), "ERROR", longDelay);
                                      reply = sendATcommand(transitPacketBuffer, "OK", "ERROR", longDelay);
                                      transitPacketBuffer = "";
                                      i = 0;      
                                      packetCounter++; 
                                    }
                                  }
                                  if(transitPacketBuffer.length() != 0){
                                    reply = sendATcommand("AT+FTPPUT=2," + String(i), "+FTPPUT: 2," + String(i), "ERROR", longDelay);
                                    if (reply == 0){
                                      reply = sendATcommand(transitPacketBuffer, "OK", "ERROR", longDelay); 
                                      packetCounter++;
                                    }   
                                  }
                                  reply = sendATcommand("AT+FTPPUT=2,0", "+FTPPUT: 1,", "ERROR", longDelay); 
                                  imageFile.close();                                   
                                  if(packetCounter!=numberOfPackets){
                                    reply = 2;
                                    pDBG("#----------"+String(picName)+"-->>Packet Count Error: ");
                                    pDBGln(numberOfPackets-packetCounter);                             
                                  }else{
                                    pDBG(String(picName));         
                                    pDBG(", SDsize: ");
                                    pDBG(picSize);
                                    pDBGln(" bytes");
                                  }
                                  if(bytesTransfered!=fileSize){
                                    pDBG("-------->>>Bytes Transfer Failure: ");
                                    pDBG(String(picName));
                                    pDBG(" Filesize: ");
                                    pDBG(fileSize);
                                    pDBG("  -Transfered: ");
                                    pDBGln(bytesTransfered);
                                  }
                                }
                              }
                            }
                          }
                        }
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }
  return reply;
}

byte sendATcommand(String ATcommand, String answer1, String answer2, unsigned int timeout){
  byte reply = 1;
  String content = "";
  char character;
 
  //Clean the modem input buffer
  while(SerialAT.available()>0){
    SerialAT.read();
  }
  //Send the atcommand to the modem
  SerialAT.println(ATcommand);
  delay(100);
  unsigned int timeprevious = millis();
  while((reply == 1) && ((millis() - timeprevious) < timeout)){
    while(SerialAT.available()>0) {
      character = SerialAT.read();
      content.concat(character);
      pDBG(character);
      delay(15);
    }

    //Stop reading conditions
    if (content.indexOf(answer1) != -1){
      reply = 0;
    }else if(content.indexOf(answer2) != -1){
      reply = 2;
    }else{
      //Nothing to do...
    }  
    if (content.indexOf("+FTPSIZE:") != -1){      //+FTPSIZE: 1,0,2324
      imgSizeFTP = content.substring(22).toInt();     
    }else if(content.indexOf("+CCLK:") != -1){
      GSMTimeStamp  = content.substring(10,12); 
      GSMTimeStamp += content.substring(13,15);
      GSMTimeStamp += content.substring(16,18);
      GSMTimeStamp += content.substring(19,21); 
      GSMTimeStamp += content.substring(22,24);
      GSMTimeStamp += content.substring(25,27);
      //pDBGln(GSMTimeStamp);
    }
  }
  pDBG(ATcommand);
  pDBG(" Response: ");
  pDBGln(content);
  return reply;
}

Thanks in advance
Paulo