RTIO Underflow error

Charu

I am trying to run this code and coming across a RTIO underflow error. I looked at relevent discussions from @JohnDVill @qllopg @jqt3 and found that break_realtime() overcomes that error usually and also rebooting/restarting. But I am still getting the error:

def run(self):
       # Code for the gradient ascent and optimization
       print("Running self")
       target_state = np.array([1, 0, 0])
       max_iterations = 10
       threshold = 0.01
       params0 = (np.random.rand(4) - 0.5) * 4000
       print("Entering while loop now")
       while self.read_inner(params0, target_state) < 3.15e-07:
           print("read_inner:", self.read_inner(params0, target_state))
           print("params0:", params0)
           print("target_state:", target_state)
           params0 = (np.random.rand(4) - 0.5) * 4000
       print("Exited while loop")
       grad_step = 50
       p = params0.copy()
       pgrad = params0.copy()
       p_history = p
       print("Entering read_inner now")
       f_history = self.read_inner(p, target_state)
       i = 0
       diff = 1
       print("Entering for loop now")
    #    while (i < max_iterations) and (diff > threshold):
       self.core.break_realtime()
       for i in tqdm(range(max_iterations)):
           if diff < threshold:
               break
           for j in range(4):
               if np.random.rand() > 0.5:
                   pgrad[j] += grad_step
           pgrad = self.grad_func(p, pgrad, target_state, 100)
           p = pgrad.copy()
           p_history = np.vstack((p_history, p))
           f_history = np.vstack((f_history, self.read_inner(p, target_state)))
           i += 1
           diff = np.absolute(f_history[-1] - 1)
           self.core.break_realtime()
           for ch, voltage in enumerate(p):
               self.set_zotino_voltage(ch+1, voltage)
           self.core.break_realtime()
           power = self.pm.read# reading power from pm
           print("Power:", power)
   def set_zotino_voltage(self, channel, voltage):
       self.zotino0.set_dac_mu([int(channel)], [int(self.volts_to_mu(voltage))])
       self.core.break_realtime()
   def volts_to_mu(self, voltage):
       return int(voltage) # converting voltage to Zotino Machine Units(MU)
   def read_inner(self, params, target_state):
       for ch in range(len(params)):
           self.epc._ask(f'V{ch+1}={params[ch]}')
       time.sleep(0.05)
       return self.pm.read

Th error is:

Core Device Traceback:
Traceback (most recent call first):
  File "runtime/src/rtio_csr.rs", line 74, column 10, in (Rust function)
    <unknown>
             ^
  File "<artiq>/coredevice/spi2.py", line 225, in ... artiq.coredevice.spi2.SPIMaster.write<artiq.coredevice.spi2.SPIMaster>(...) (RA=+0x2f4)
    rtio_output((self.channel << 8) | SPI_DATA_ADDR, data)
  File "<artiq>/coredevice/ad53xx.py", line 276, in ... artiq.coredevice.ad53xx.AD53xx.write_dac_mu<artiq.coredevice.zotino.Zotino>(...) (inlined)
    self.bus.write(
  File "<artiq>/coredevice/ad53xx.py", line 329, in artiq.coredevice.ad53xx.AD53xx.set_dac_mu(..., ...) (inlined)
    self.write_dac_mu(channels[i], values[i])
artiq.coredevice.exceptions.RTIOUnderflow(0): RTIO underflow at 705055793070244 mu, channel 8, slack -265237316 mu

End of Core Device Traceback

Traceback (most recent call last):
  File "/nix/store/ky5j6106mjc6qgf1l4607z3bpg3yclgl-python3.9-artiq-7.8173.ff97675/bin/.artiq_run-wrapped", line 9, in <module>
    sys.exit(main())
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/frontend/artiq_run.py", line 224, in main
    return run(with_file=True)
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/frontend/artiq_run.py", line 210, in run
    raise exn
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/frontend/artiq_run.py", line 203, in run
    exp_inst.run()
  File "/home/cjain/quant-net-dev-control/dev_control/polar_ctrl.py", line 120, in run
    self.set_zotino_voltage(ch+1, voltage)
  File "/home/cjain/quant-net-dev-control/dev_control/polar_ctrl.py", line 125, in set_zotino_voltage
    self.zotino0.set_dac_mu([int(channel)], [int(self.volts_to_mu(voltage))])
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/language/core.py", line 54, in run_on_core
    return getattr(self, arg).run(run_on_core, ((self,) + k_args), k_kwargs)
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/coredevice/core.py", line 140, in run
    self._run_compiled(kernel_library, embedding_map, symbolizer, demangler)
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/coredevice/core.py", line 130, in _run_compiled
    self.comm.serve(embedding_map, symbolizer, demangler)
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/coredevice/comm_kernel.py", line 716, in serve
    self._serve_exception(embedding_map, symbolizer, demangler)
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/coredevice/comm_kernel.py", line 698, in _serve_exception
    raise python_exn
artiq.coredevice.exceptions.RTIOUnderflow: RTIO underflow at 705055793070244 mu, channel 8, slack -265237316 mu

Please let me know what I can do, thanks and regards.

Charu

I changed the code by including parallel blocks:

def run(self):
        self.core.reset()
        # Code for the gradient ascent and optimization
        print("Running self")
        target_state = np.array([1, 0, 0])
        max_iterations = 10
        threshold = 0.01
        params0 = (np.random.rand(4) - 0.5) * 4000
        print("Entering while loop now")
        while self.read_inner(params0, target_state) < 2.5e-07:
            print("read_inner:", self.read_inner(params0, target_state))
            print("params0:", params0)
            print("target_state:", target_state)
            params0 = (np.random.rand(4) - 0.5) * 4000
        print("Exited while loop")
        grad_step = 50
        p = params0.copy()
        pgrad = params0.copy()
        p_history = p
        print("Entering read_inner now")
        f_history = self.read_inner(p, target_state)
        i = 0
        diff = 1
        print("Entering for loop now")
    #    while (i < max_iterations) and (diff > threshold):
        #self.core.break_realtime()
        for i in tqdm(range(max_iterations)):
            if diff < threshold:
                break
            for j in range(4):
                if np.random.rand() > 0.5:
                    pgrad[j] += grad_step
            pgrad = self.grad_func(p, pgrad, target_state, 100)
            p = pgrad.copy()
            p_history = np.vstack((p_history, p))
            f_history = np.vstack((f_history, self.read_inner(p, target_state)))
            i += 1
            diff = np.absolute(f_history[-1] - 1)
            self.core.break_realtime()
        #    ch = np.random.randint(4)
        #    voltage = p[ch - 1]
            
            try:
                with parallel:
                    for ch, voltage in enumerate(p):
                        self.set_zotino_voltage(ch+1, voltage)
            except RTIOUnderflow:
                with parallel:
                    for ch, voltage in enumerate(p):
                        self.set_zotino_voltage(ch+1, voltage)
            power = self.pm.read# reading power from pm
            print("Power:", power)
    @kernel
    def set_zotino_voltage(self, channel, voltage):
        self.core.break_realtime()
        self.zotino0.set_dac_mu([int(channel)], [int(self.volts_to_mu(voltage))])
    def volts_to_mu(self, voltage):
        return int(voltage) # converting voltage to Zotino Machine Units(MU)
    def read_inner(self, params, target_state):
        for ch in range(len(params)):
            self.epc.write_v(ch+1, params[ch]) 
        return self.pm.read

And I am getting this error:

^CTraceback (most recent call last):
  File "/nix/store/ky5j6106mjc6qgf1l4607z3bpg3yclgl-python3.9-artiq-7.8173.ff97675/bin/.artiq_run-wrapped", line 9, in <module>
    sys.exit(main())
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/frontend/artiq_run.py", line 224, in main
    return run(with_file=True)
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/frontend/artiq_run.py", line 203, in run
    exp_inst.run()
  File "/home/cjain/quant-net-dev-control/dev_control/polar_ctrl.py", line 94, in run
    while self.read_inner(params0, target_state) < 0.7:
  File "/home/cjain/quant-net-dev-control/dev_control/polar_ctrl.py", line 145, in read_inner
    self.epc.write_v(ch+1, params[ch]) 
  File "/home/cjain/quant-net-dev-control/dev_control/polar_ctrl.py", line 41, in write_v
    self.ser.write(('V%d,%d\r\n'%(ch, value)).encode())
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/serial/serialposix.py", line 640, in write
    abort, ready, _ = select.select([self.pipe_abort_write_r], [self.fd], [], None)
KeyboardInterrupt

root@quantumnode:/home/cjain/quant-net-dev-control/artiq# sudo /nix/store/xsin7a4gbxhlmj6zmy3gnvm9c2ddv9zp-artiq-env/bin/artiq_run /home/cjain/quant-net-dev-control/dev_control/polar_ctrl.py
2.54217184e-07
Lambda: 800.0
Current config: POW
Setting average to 10 ...
PD diode bandwidth: 1
Setting bandwidth to High
PD diode bandwidth: 0
Setting bandwidth to Low
PD diode bandwidth: 1
Running self
Entering while loop now
Exited while loop
Entering read_inner now
Entering for loop now
  0%|                                                    | 0/10 [00:00<?, ?it/s]
Traceback (most recent call last):
  File "/nix/store/ky5j6106mjc6qgf1l4607z3bpg3yclgl-python3.9-artiq-7.8173.ff97675/bin/.artiq_run-wrapped", line 9, in <module>
    sys.exit(main())
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/frontend/artiq_run.py", line 224, in main
    return run(with_file=True)
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/frontend/artiq_run.py", line 210, in run
    raise exn
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/frontend/artiq_run.py", line 203, in run
    exp_inst.run()
  File "/home/cjain/quant-net-dev-control/dev_control/polar_ctrl.py", line 128, in run
    with parallel:
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/language/core.py", line 242, in __enter__
    _time_manager.enter_parallel()
  File "/nix/store/8071a0dzm1x8fba780nqr9r2zf9ppkxi-python3-3.9.16-env/lib/python3.9/site-packages/artiq/language/core.py", line 198, in _not_implemented
    raise NotImplementedError(
NotImplementedError: Attempted to interpret kernel without a time manager

What am I doing wrong?

steine01

As far as I know, parallel only works in an @kernel function and you use it in run, which is not declared as an @kernel function. In the first example, you got an RTIOUnderflow with a lot of slack missing, which points to the fact, that you tried to call an @kernel function (like self.zotino0.set_dac_mu) outside of an @kernel context. And this is what was your main error there

def set_zotino_voltage(self, channel, voltage):
       self.zotino0.set_dac_mu([int(channel)], [int(self.volts_to_mu(voltage))])
       self.core.break_realtime()

is missing the @kernel decorator

@kernel
def set_zotino_voltage(self, channel, voltage):
       self.zotino0.set_dac_mu([int(channel)], [int(self.volts_to_mu(voltage))])
       self.core.break_realtime()

You added it in the second example, but also included the parallel statements, which do not work outside of an @kernel function, like I mentioned in the beginning.

Charu

Thank you so much!