PCAN API Java Thread termination Hangs

[btshrewsbury]

Hello,

We are having issues shutting down our Java programs when using the PCANBasic API. We are using PCAN's PCI Quad CAN and M.2 devices on Ubuntu 20.04.2 LTS (Different Computers). We are running with the PREEMPT RT Patch and using PCAN-Basic_Linux-4.5.2 and peak-linux-driver-8.13.0. I think we are having a similar issue mentioned here: viewtopic.php?f=119&t=5980&p=13945&hili ... ize#p13945.


When we press ctrl+c the program stops some of the other java threads, but the PCAN Native Library seems to hang and the program never dies. If we call Uninitialize it also just hangs in the native layer. Currently the only two ways we can successfully stop the program is to Ctrl+Z then pkill the java process or call System.exit(0). I've attached the code below.

Please let me know if you have any ideas or need me to run any tests.
Thank you!
brandon

Code: Select all

package us.ihmc.teststands;

import gnu.trove.map.hash.TIntObjectHashMap;
import peak.can.basic.*;
import us.ihmc.commons.Conversions;
import us.ihmc.realtime.*;
import us.ihmc.robotDataLogger.YoVariableServer;
import us.ihmc.robotDataLogger.logger.DataServerSettings;
import us.ihmc.robotics.math.functionGenerator.YoFunctionGenerator;
import us.ihmc.tMotorCore.CANMessages.TMotorCANReplyMessage;
import us.ihmc.tMotorCore.TMotor;
import us.ihmc.tMotorCore.TMotorLowLevelController;
import us.ihmc.tMotorCore.TMotorVersion;
import us.ihmc.yoVariables.registry.YoRegistry;
import us.ihmc.yoVariables.variable.YoBoolean;
import us.ihmc.yoVariables.variable.YoDouble;
import us.ihmc.yoVariables.variable.YoInteger;
import us.ihmc.yoVariables.variable.YoLong;

import java.util.HashMap;

import static peak.can.basic.TPCANStatus.PCAN_ERROR_QRCVEMPTY;

public class TMotorTestBed extends RealtimeThread
{
   private final YoRegistry registry = new YoRegistry(getClass().getSimpleName());
   private final YoVariableServer yoVariableServer;

   private static final double DT = 0.001;
   private static final MonotonicTime period = new MonotonicTime(0, Conversions.secondsToNanoseconds(DT));

   private final static PriorityParameters controllerPriority = new PriorityParameters(PriorityParameters.getMaximumPriority() - 5);

   private final YoLong tickStartTimeInNanos = new YoLong("tickStartTimeInNanos", registry);
   private final YoLong doControlTime = new YoLong("doControlTime", registry);
   private final YoLong computeTime = new YoLong("computeTime", registry);
   private final YoLong canReadTime = new YoLong("canReadTime", registry);
   private final YoLong canWriteTime = new YoLong("canWriteTime", registry);
   private final YoLong effectiveDT = new YoLong("effectiveDT", registry);
   private final YoDouble yoTime = new YoDouble("yoTime", registry);

   private final YoLong readErrorCounter = new YoLong("readErrorCounter", registry);
   private final YoLong writeErrorCounter = new YoLong("writeErrorCounter", registry);
   private final YoBoolean running = new YoBoolean("running", registry);

   // motors in CAN bus
   private final TIntObjectHashMap<TMotor> motors = new TIntObjectHashMap<>();
   private int[] motorIDs;
   private static final int RIGHT_HIP_CAN_ID = 2;
   private static final int KNEE_CAN_ID = 11;
   private final HashMap<TMotor, TMotorLowLevelController> motorControllers = new HashMap<>();
   private final YoFunctionGenerator functionGenerator;

   // CAN-related goodies
   private PCANBasic can = new PCANBasic();
   private final TPCANHandle channel = TPCANHandle.PCAN_PCIBUS1;
   private final TPCANMsg receivedMsg = new TPCANMsg();
   private TPCANStatus status = null;

   // specialized YoVariables
   private final YoBoolean enableCANMsgs = new YoBoolean("enableCANMsgs", registry);
   private final YoBoolean resetCounters = new YoBoolean("resetCounters", registry);

   // debug
   private final YoInteger messagesInReadBus = new YoInteger("messagesInBus", registry);

   public TMotorTestBed(YoVariableServer yoVariableServer)
   {
      super(controllerPriority, new PeriodicParameters(period));

      CPUDMALatency.setLatency(0);

      initializeCAN();

      this.yoVariableServer = yoVariableServer;
      yoVariableServer.setMainRegistry(registry, null);

      TMotor hipMotor = new TMotor(RIGHT_HIP_CAN_ID, "hipMotor", TMotorVersion.AK109, DT, registry);
      motors.put(hipMotor.getID(), hipMotor);
      TMotorLowLevelController hipController = new TMotorLowLevelController("hipController", hipMotor, registry);
      hipController.setUnsafeOutputSpeed(16.0);
      motorControllers.put(hipMotor, hipController);
      functionGenerator = new YoFunctionGenerator("functionGenerator", yoTime, registry);
      functionGenerator.setAlphaForSmoothing(0.99);

      TMotor kneeMotor = new TMotor(KNEE_CAN_ID, "kneeMotor", TMotorVersion.AK109, DT, registry);
      motors.put(kneeMotor.getID(), kneeMotor);
      TMotorLowLevelController kneeController = new TMotorLowLevelController("kneeController", kneeMotor, registry);
      kneeController.setUnsafeOutputSpeed(16.0);
      motorControllers.put(kneeMotor, kneeController);

      motorIDs = motors.keys();

      receivedMsg.setLength((byte) 6);
      enableCANMsgs.set(true);
      running.set(true);
   }

   private void initializeCAN()
   {
      if (!can.initializeAPI())
      {
         System.out.println("Unable to initialize the API");
         System.exit(0);
      }
      else
      {
         System.out.println("CAN API has been initialized");
      }
      status = can.Initialize(channel, TPCANBaudrate.PCAN_BAUD_1M, TPCANType.PCAN_TYPE_NONE, 0, (short) 0);
   }

   @Override
   public void run()
   {
      long controllerStartTime = System.nanoTime();
      while (running.getBooleanValue())
      {
         effectiveDT.set(System.nanoTime() - tickStartTimeInNanos.getLongValue());
         tickStartTimeInNanos.set(System.nanoTime());

         yoTime.set(Conversions.nanosecondsToSeconds(tickStartTimeInNanos.getLongValue() - controllerStartTime));

         if(resetCounters.getBooleanValue())
         {
            readErrorCounter.set(0);
            writeErrorCounter.set(0);
            resetCounters.set(false);
         }

         if (enableCANMsgs.getBooleanValue())
         {

            long canReadStartTime = System.nanoTime();
            read();
            canReadTime.set(System.nanoTime() - canReadStartTime);

            long computeStartTime = System.nanoTime();
            compute();
            computeTime.set(System.nanoTime() - computeStartTime);

            long canWriteStartTime = System.nanoTime();
            write();
            canWriteTime.set(System.nanoTime() - canWriteStartTime);


            // TODO safe sentinel
         }
         // wait to free sent queue
         yoVariableServer.update(tickStartTimeInNanos.getLongValue());
         doControlTime.set(System.nanoTime() - tickStartTimeInNanos.getLongValue());
         waitForNextPeriod();
      }
   }

   private void read()
   {
      TPCANStatus readStatus = can.Read(channel, receivedMsg, null);

      messagesInReadBus.set(0);
      while(readStatus != PCAN_ERROR_QRCVEMPTY)
      {
         if (readStatus == TPCANStatus.PCAN_ERROR_OK)
         {
            int id = TMotorCANReplyMessage.getID(receivedMsg);
            if(motors.containsKey(id))
               motors.get(id).read(receivedMsg);
            messagesInReadBus.increment();
         }
         else
         {
            readErrorCounter.increment();
         }
         readStatus = can.Read(channel, receivedMsg, null);
      }
   }

   private void compute()
   {
      for(int id = 0; id < motorIDs.length; id++)
      {
         motorControllers.get(motors.get(motorIDs[id])).setDesiredPosition(functionGenerator.getValue());
         motorControllers.get(motors.get(motorIDs[id])).setDesiredVelocity(functionGenerator.getValueDot());
         motorControllers.get(motors.get(motorIDs[id])).setDesiredTorque(0.0);

         motorControllers.get(motors.get(motorIDs[id])).doControl();
      }
   }

   private void write()
   {
      for(int id = 0; id < motorIDs.length; id++)
      {
         TPCANMsg motorCommand = motors.get(motorIDs[id]).getCommandedMsg();
         status = can.Write(channel, motorCommand);
         if (status != TPCANStatus.PCAN_ERROR_OK)
         {
            writeErrorCounter.increment();
         }
      }
   }



   public static void main(String[] args)
   {
      YoVariableServer yoVariableServer = new YoVariableServer(TMotorTestBed.class, null, new DataServerSettings(true), DT);
      TMotorTestBed testbed = new TMotorTestBed(yoVariableServer);

      yoVariableServer.start();

      testbed.start();
      testbed.join();

      yoVariableServer.close();

   }
}

[F.Vergnaud]

Hello Brandon,

My first guess would be that there is some deadlock with the Java-threads (there are no internal threads created within PCANBasic Linux API, so there is no obvious reason why the Uninitialize function would block).
Where/when do you call "can.Uninitialize" in your code so that it hangs? You are calling initializeCAN in TMotorTestBed constructor which is run in the thread Main, I'd suggest to have PCANBasic channels uninitialized in the same thread.

Have you checked that calls to "can.Initialize" and "can.Uninitialize" perform correctly within your main function? It would confirm that it is not an unknown side effect of the PREEMPT RT Patch.

[F.Vergnaud]

Please note, I've checked your use-case with a minimal Java sample (with and without RT Patch) with latest PCAN driver 8.14.
But I could not reproduce your issue, you'll find below the sample.

Code: Select all

package src;

import peak.can.basic.*;

public class Example extends java.lang.Thread 
{
   // CAN-related goodies
   private PCANBasic apiPcanBasic = new PCANBasic();

	public static final TPCANHandle PCAN_CHANNEL = TPCANHandle.PCAN_USBBUS1;
	public static final TPCANBaudrate PCAN_BAUDRATE = TPCANBaudrate.PCAN_BAUD_500K;
	public static final int PCAN_LENGTH = 8;

   public Example()
   {
      // Initialize the API
      if (!apiPcanBasic.initializeAPI()) {
         System.err.println("Error initializing API");
         return;
      }
      
      // Bus initialization
      System.out.println("Initializing channel " + PCAN_CHANNEL + " @ " + PCAN_BAUDRATE + "...");
      TPCANStatus status = apiPcanBasic.Initialize(PCAN_CHANNEL, PCAN_BAUDRATE, TPCANType.PCAN_TYPE_NONE, 0, (short)0);
      if (status != TPCANStatus.PCAN_ERROR_OK) {
         System.err.println(" -> failed to initialize channel: " + status);
         return;
      }
   }

   /**
    * Starts thread process
    */
   public void run()
   {      
      TPCANStatus status;
      
      // Reading loop sample
      boolean doLoop = true;
      long timeStart = System.currentTimeMillis();
      long timestampLast = 0;
      long testDuration = 5000;
      System.out.println("Starting reading loop for " + testDuration + "ms...");
      while (doLoop) {
         TPCANMsg msg = new TPCANMsg();
         TPCANTimestamp time = new TPCANTimestamp();
         status = apiPcanBasic.Read(PCAN_CHANNEL, msg, time);
         
         if (status == TPCANStatus.PCAN_ERROR_OK) {
            long diff = time.getMillis() - timestampLast;
            timestampLast = time.getMillis();
            System.out.format(" * Read CAN message 0x%04x @%12d, %+12d.%n", msg.getID(), timestampLast, diff);
         }
         else if (status != TPCANStatus.PCAN_ERROR_QRCVEMPTY) {
            System.err.println("Error receiving data from channel: " + status);				
         }
         doLoop &= ((System.currentTimeMillis() - timeStart) < testDuration);
      }
      System.out.println("Completed reading loop, exiting...");

      // Bus uninitialization
      status = apiPcanBasic.Uninitialize(PCAN_CHANNEL);
      if (status != TPCANStatus.PCAN_ERROR_OK) {
         System.err.println(" -> failed to uninitialize channel: " + status);
      }
   }

   public static void main(String[] args)
   {
      // Create and start ReadMessage Thread
      Example canReadThread = new Example();
      canReadThread.start();
      try {
         canReadThread.join();
      } catch (InterruptedException e) {
         e.printStackTrace();
      }
   }
}

[F.Vergnaud]

Hello Brandon,

I have to apologize, there is indeed an issue with CTRL-C interruption and the linux version of libpcanbasic_jni (<=4.5.0, library is catching SIGINT signals).
Although I didn't find a link between this problem and your hanging within Uninitialize function, if an application also catches SIGINT signals there can be unforseen side-effects.

A fix is ready and will be available in the next release. If you want to test a beta version, please send an email to linux[at]peak-system.com making reference to this post.

[btshrewsbury]

Hi Fabrice,

We are happy to try out the beta version. For the uninitialize call, we do call them from the same thread. I was only calling uninitialize out of hope that it would clean things up nicely between runs, but it hasn't seemed to effect reliability not calling it. Do you think it is a bug worth investigating on our end or can we ignore it?

thank you!
brandon