Skip to content

Commit

Permalink
Merge pull request #52 from dji-sdk/bugfix/fix_widget_speaker_crash_bug
Browse files Browse the repository at this point in the history
FIX: fix the widget speaker crash bug and usb bulk issue on cxx platform
  • Loading branch information
dji-dev committed May 30, 2022
2 parents 22e82a2 + 4b6b961 commit 3d8e2b5
Show file tree
Hide file tree
Showing 20 changed files with 113 additions and 57 deletions.
2 changes: 1 addition & 1 deletion psdk_lib/include/dji_version.h
Expand Up @@ -37,7 +37,7 @@ extern "C" {
#define DJI_VERSION_MINOR 1 /*!< DJI SDK minor version num, when add functionality in a backwards compatible manner changes. Range from 0 to 99. */
#define DJI_VERSION_MODIFY 0 /*!< DJI SDK modify version num, when have backwards compatible bug fixes changes. Range from 0 to 99. */
#define DJI_VERSION_BETA 0 /*!< DJI SDK version beta info, release version will be 0, when beta version release changes. Range from 0 to 255. */
#define DJI_VERSION_BUILD 1503 /*!< DJI SDK version build info, when jenkins trigger build changes. Range from 0 to 65535. */
#define DJI_VERSION_BUILD 1509 /*!< DJI SDK version build info, when jenkins trigger build changes. Range from 0 to 65535. */

/* Exported types ------------------------------------------------------------*/

Expand Down
17 changes: 9 additions & 8 deletions psdk_lib/include/dji_widget.h
Expand Up @@ -70,26 +70,27 @@ typedef enum {
* @brief Switch widget speaker work mode.
*/
typedef enum {
DJI_WIDGET_SPEAKER_WORK_MODE_TTS,
DJI_WIDGET_SPEAKER_WORK_MODE_VOICE,
DJI_WIDGET_SPEAKER_WORK_MODE_TTS = 0,
DJI_WIDGET_SPEAKER_WORK_MODE_VOICE = 1,
} E_DjiWidgetSpeakerWorkMode;

/**
* @brief Switch widget speaker play mode.
*/
typedef enum {
DJI_WIDGET_SPEAKER_PLAY_MODE_SINGLE_PLAY,
DJI_WIDGET_SPEAKER_PLAY_MODE_LOOP_PLAYBACK,
DJI_WIDGET_SPEAKER_PLAY_MODE_SINGLE_PLAY = 0,
DJI_WIDGET_SPEAKER_PLAY_MODE_LOOP_PLAYBACK = 1,
} E_DjiWidgetSpeakerPlayMode;

/**
* @brief Switch widget speaker state.
*/
typedef enum {
DJI_WIDGET_SPEAKER_STATE_IDEL,
DJI_WIDGET_SPEAKER_STATE_TRANSMITTING,
DJI_WIDGET_SPEAKER_STATE_PLAYING,
DJI_WIDGET_SPEAKER_STATE_ERROR,
DJI_WIDGET_SPEAKER_STATE_IDEL = 0,
DJI_WIDGET_SPEAKER_STATE_TRANSMITTING = 1,
DJI_WIDGET_SPEAKER_STATE_PLAYING = 2,
DJI_WIDGET_SPEAKER_STATE_ERROR = 3,
DJI_WIDGET_SPEAKER_STATE_IN_TTS_CONVERSION = 4,
} E_DjiWidgetSpeakerState;

/**
Expand Down
Binary file modified psdk_lib/lib/aarch64-himix100-linux-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/aarch64-linux-android-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/aarch64-linux-gnu-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-himix100-linux-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-himix200-linux-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-hisiv300-linux-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-hisiv400-linux-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-hisiv500-linux-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-hisiv600-linux-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-linux-androideabi-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-linux-gnueabi-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-linux-gnueabihf-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/arm-none-eabi-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/armcc_cortex-m4/libpayload.lib
Binary file not shown.
Binary file modified psdk_lib/lib/x86_64-linux-gnu-gcc/libpayloadsdk.a
Binary file not shown.
Binary file modified psdk_lib/lib/xtensa-esp32-elf-gcc/libpayloadsdk.a
Binary file not shown.
25 changes: 9 additions & 16 deletions samples/sample_c++/platform/linux/manifold2/hal/hal_usb_bulk.c
Expand Up @@ -30,7 +30,10 @@
#define LINUX_USB_BULK_TRANSFER_TIMEOUT_MS (50)
#define LINUX_USB_BULK_TRANSFER_WAIT_FOREVER (-1)

#define LINUX_USB_BULK_DEV1 "/dev/usb-ffs/bulk"
#define LINUX_USB_BULK_EP_OUT "/dev/usb-ffs/bulk/ep1"
#define LINUX_USB_BULK_EP_IN "/dev/usb-ffs/bulk/ep2"
#define LINUX_USB_PID (0x0955)
#define LINUX_USB_VID (0x7020)

/* Private types -------------------------------------------------------------*/
typedef struct {
Expand Down Expand Up @@ -80,12 +83,12 @@ T_DjiReturnCode HalUsbBulk_Init(T_DjiHalUsbBulkInfo usbBulkInfo, T_DjiUsbBulkHan
((T_HalUsbBulkObj *) *usbBulkHandle)->handle = handle;
memcpy(&((T_HalUsbBulkObj *) *usbBulkHandle)->usbBulkInfo, &usbBulkInfo, sizeof(usbBulkInfo));

((T_HalUsbBulkObj *) *usbBulkHandle)->ep1 = open("/dev/usb-ffs/bulk/ep1", O_RDWR);
((T_HalUsbBulkObj *) *usbBulkHandle)->ep1 = open(LINUX_USB_BULK_EP_OUT, O_RDWR);
if (((T_HalUsbBulkObj *) *usbBulkHandle)->ep1 < 0) {
return DJI_ERROR_SYSTEM_MODULE_CODE_SYSTEM_ERROR;
}

((T_HalUsbBulkObj *) *usbBulkHandle)->ep2 = open("/dev/usb-ffs/bulk/ep2", O_RDWR);
((T_HalUsbBulkObj *) *usbBulkHandle)->ep2 = open(LINUX_USB_BULK_EP_IN, O_RDWR);
if (((T_HalUsbBulkObj *) *usbBulkHandle)->ep2 < 0) {
return DJI_ERROR_SYSTEM_MODULE_CODE_SYSTEM_ERROR;
}
Expand Down Expand Up @@ -177,19 +180,9 @@ T_DjiReturnCode HalUsbBulk_ReadData(T_DjiUsbBulkHandle usbBulkHandle, uint8_t *b

T_DjiReturnCode HalUsbBulk_GetDeviceInfo(T_DjiHalUsbBulkDeviceInfo *deviceInfo)
{
//attention: need confirm your usb config in device mode.
deviceInfo->vid = 0x0955;
deviceInfo->pid = 0x7020;

deviceInfo->bulkChannelNum = 2;

deviceInfo->channelInfo[0].interfaceNum = 0;
deviceInfo->channelInfo[0].endPointIn = 0x01;
deviceInfo->channelInfo[0].endPointOut = 0x81;

deviceInfo->channelInfo[1].interfaceNum = 0;
deviceInfo->channelInfo[1].endPointIn = 0x02;
deviceInfo->channelInfo[1].endPointOut = 0x82;
//attention: this interface only be called in usb device mode.
deviceInfo->vid = LINUX_USB_VID;
deviceInfo->pid = LINUX_USB_PID;

return DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS;
}
Expand Down
126 changes: 94 additions & 32 deletions samples/sample_c/module_sample/widget/test_widget_speaker.c
Expand Up @@ -43,13 +43,14 @@
/* Private constants ---------------------------------------------------------*/
#define WIDGET_SPEAKER_TASK_STACK_SIZE (2048)

/*! Attention: replace your audio device name here. */
#define WIDGET_SPEAKER_USB_AUDIO_DEVICE_NAME "alsa_output.usb-C-Media_Electronics_Inc._USB_Audio_Device-00.analog-stereo"

#define WIDGET_SPEAKER_AUDIO_OPUS_FILE_NAME "test_audio.opus"
#define WIDGET_SPEAKER_AUDIO_PCM_FILE_NAME "test_audio.pcm"

#define WIDGET_SPEAKER_TTS_FILE_NAME "test_tts.txt"
#define WIDGET_SPEAKER_TTS_OUTPUT_FILE_NAME "tts_sample.wav"
#define WIDGET_SPEAKER_TTS_OUTPUT_FILE_NAME "tts_audio.wav"
#define WIDGET_SPEAKER_TTS_FILE_MAX_SIZE (3000)

/* The frame size is hardcoded for this sample code but it doesn't have to be */
Expand All @@ -60,7 +61,8 @@
#define WIDGET_SPEAKER_AUDIO_OPUS_DECODE_FRAME_SIZE (160)

/* The speaker initialization parameters */
#define WIDGET_SPEAKER_DEFAULT_VOLUME (50)
#define WIDGET_SPEAKER_DEFAULT_VOLUME (30)
#define EKHO_INSTALLED (1)

/* Private types -------------------------------------------------------------*/

Expand All @@ -70,10 +72,12 @@ static T_DjiMutexHandle s_speakerMutex = {0};
static T_DjiWidgetSpeakerState s_speakerState = {0};
static T_DjiTaskHandle s_widgetSpeakerTestThread;

static FILE *audioFile = NULL;
static FILE *ttsFile = NULL;
static FILE *s_audioFile = NULL;
static FILE *s_ttsFile = NULL;
static bool s_isDecodeFinished = true;

/* Private functions declaration ---------------------------------------------*/
static void SetSpeakerState(E_DjiWidgetSpeakerState speakerState);
static T_DjiReturnCode GetSpeakerState(T_DjiWidgetSpeakerState *speakerState);
static T_DjiReturnCode SetWorkMode(E_DjiWidgetSpeakerWorkMode workMode);
static T_DjiReturnCode GetWorkMode(E_DjiWidgetSpeakerWorkMode *workMode);
Expand Down Expand Up @@ -174,7 +178,6 @@ static uint32_t DjiTest_GetVoicePlayProcessId(void)

ret = fscanf(fp, "%u", &pid);
if (ret <= 0) {
USER_LOG_ERROR("get pid error.");
pid = 0;
goto out;
}
Expand Down Expand Up @@ -271,6 +274,7 @@ static T_DjiReturnCode DjiTest_DecodeAudioData(void)
fclose(fout);

USER_LOG_INFO("Decode Finished...");
s_isDecodeFinished = true;

#endif
return EXIT_SUCCESS;
Expand All @@ -281,6 +285,7 @@ static T_DjiReturnCode DjiTest_PlayAudioData(void)
char cmdStr[128];

memset(cmdStr, 0, sizeof(cmdStr));
USER_LOG_INFO("Start Playing...");

snprintf(cmdStr, sizeof(cmdStr), "ffplay -nodisp -autoexit -ar 16000 -ac 1 -f s16le -i %s 2>/dev/null",
WIDGET_SPEAKER_AUDIO_PCM_FILE_NAME);
Expand Down Expand Up @@ -315,8 +320,22 @@ static T_DjiReturnCode DjiTest_PlayTtsData(void)

memset(cmdStr, 0, sizeof(cmdStr));

SetSpeakerState(DJI_WIDGET_SPEAKER_STATE_IN_TTS_CONVERSION);

#ifdef EKHO_INSTALLED
/*! Attention: you can use other tts opensource function to convert txt to speech, example used ekho v7.5 */
snprintf(cmdStr, sizeof(cmdStr), " ekho %s -s 20 -p 20 -a 100", data);
#else
snprintf(cmdStr, sizeof(cmdStr), "tts_offline_sample '%s' %s", data,
WIDGET_SPEAKER_TTS_OUTPUT_FILE_NAME);
DjiUserUtil_RunSystemCmd(cmdStr);

SetSpeakerState(DJI_WIDGET_SPEAKER_STATE_PLAYING);
USER_LOG_INFO("Start TTS Playing...");
memset(cmdStr, 0, sizeof(cmdStr));
snprintf(cmdStr, sizeof(cmdStr), "ffplay -nodisp -autoexit -ar 16000 -ac 1 -f s16le -i %s 2>/dev/null",
WIDGET_SPEAKER_TTS_OUTPUT_FILE_NAME);
#endif

return DjiUserUtil_RunSystemCmd(cmdStr);
}
Expand Down Expand Up @@ -359,6 +378,7 @@ static T_DjiReturnCode DjiTest_CheckFileMd5Sum(const char *path, uint8_t *buf, u

if (size == sizeof(md5Sum)) {
if (memcmp(md5Sum, buf, sizeof(md5Sum)) == 0) {
USER_LOG_INFO("MD5 sum check success");
return DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS;
} else {
return DJI_ERROR_SYSTEM_MODULE_CODE_SYSTEM_ERROR;
Expand All @@ -370,6 +390,24 @@ static T_DjiReturnCode DjiTest_CheckFileMd5Sum(const char *path, uint8_t *buf, u
return DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS;
}

static void SetSpeakerState(E_DjiWidgetSpeakerState speakerState)
{
T_DjiReturnCode returnCode;
T_DjiOsalHandler *osalHandler = DjiPlatform_GetOsalHandler();

returnCode = osalHandler->MutexLock(s_speakerMutex);
if (returnCode != DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS) {
USER_LOG_ERROR("lock mutex error: 0x%08llX.", returnCode);
}

s_speakerState.state = speakerState;

returnCode = osalHandler->MutexUnlock(s_speakerMutex);
if (returnCode != DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS) {
USER_LOG_ERROR("unlock mutex error: 0x%08llX.", returnCode);
}
}

static T_DjiReturnCode GetSpeakerState(T_DjiWidgetSpeakerState *speakerState)
{
T_DjiReturnCode returnCode;
Expand Down Expand Up @@ -484,23 +522,17 @@ static T_DjiReturnCode GetPlayMode(E_DjiWidgetSpeakerPlayMode *playMode)

static T_DjiReturnCode StartPlay(void)
{
T_DjiReturnCode returnCode;
uint32_t pid;
T_DjiOsalHandler *osalHandler = DjiPlatform_GetOsalHandler();

returnCode = osalHandler->MutexLock(s_speakerMutex);
if (returnCode != DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS) {
USER_LOG_ERROR("lock mutex error: 0x%08llX.", returnCode);
return returnCode;
pid = DjiTest_GetVoicePlayProcessId();
if (pid != 0) {
DjiTest_KillVoicePlayProcess(pid);
}

osalHandler->TaskSleepMs(5);
USER_LOG_INFO("Start widget speaker play");
s_speakerState.state = DJI_WIDGET_SPEAKER_STATE_PLAYING;

returnCode = osalHandler->MutexUnlock(s_speakerMutex);
if (returnCode != DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS) {
USER_LOG_ERROR("unlock mutex error: 0x%08llX.", returnCode);
return returnCode;
}
SetSpeakerState(DJI_WIDGET_SPEAKER_STATE_PLAYING);

return DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS;
}
Expand Down Expand Up @@ -600,29 +632,38 @@ static T_DjiReturnCode ReceiveTtsData(E_DjiWidgetTransmitDataEvent event,

if (event == DJI_WIDGET_TRANSMIT_DATA_EVENT_START) {
USER_LOG_INFO("Create tts file.");
ttsFile = fopen(WIDGET_SPEAKER_TTS_FILE_NAME, "wb");
if (ttsFile == NULL) {
s_ttsFile = fopen(WIDGET_SPEAKER_TTS_FILE_NAME, "wb");
if (s_ttsFile == NULL) {
USER_LOG_ERROR("Open tts file error.");
}
if (s_speakerState.state != DJI_WIDGET_SPEAKER_STATE_PLAYING) {
SetSpeakerState(DJI_WIDGET_SPEAKER_STATE_TRANSMITTING);
}
} else if (event == DJI_WIDGET_TRANSMIT_DATA_EVENT_TRANSMIT) {
USER_LOG_INFO("Transmit tts file, offset: %d, size: %d", offset, size);
if (ttsFile != NULL) {
fseek(ttsFile, offset, SEEK_SET);
writeLen = fwrite(buf, 1, size, ttsFile);
if (s_ttsFile != NULL) {
fseek(s_ttsFile, offset, SEEK_SET);
writeLen = fwrite(buf, 1, size, s_ttsFile);
if (writeLen != size) {
USER_LOG_ERROR("Write tts file error %d", writeLen);
}
}
if (s_speakerState.state != DJI_WIDGET_SPEAKER_STATE_PLAYING) {
SetSpeakerState(DJI_WIDGET_SPEAKER_STATE_TRANSMITTING);
}
} else if (event == DJI_WIDGET_TRANSMIT_DATA_EVENT_FINISH) {
USER_LOG_INFO("Close tts file.");
if (ttsFile != NULL) {
fclose(ttsFile);
if (s_ttsFile != NULL) {
fclose(s_ttsFile);
}

returnCode = DjiTest_CheckFileMd5Sum(WIDGET_SPEAKER_TTS_FILE_NAME, buf, size);
if (returnCode != DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS) {
USER_LOG_ERROR("File md5 sum check failed");
}
if (s_speakerState.state != DJI_WIDGET_SPEAKER_STATE_PLAYING) {
SetSpeakerState(DJI_WIDGET_SPEAKER_STATE_IDEL);
}
}

return DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS;
Expand All @@ -635,24 +676,31 @@ static T_DjiReturnCode ReceiveAudioData(E_DjiWidgetTransmitDataEvent event,
T_DjiReturnCode returnCode;

if (event == DJI_WIDGET_TRANSMIT_DATA_EVENT_START) {
s_isDecodeFinished = false;
USER_LOG_INFO("Create voice file.");
audioFile = fopen(WIDGET_SPEAKER_AUDIO_OPUS_FILE_NAME, "wb");
if (audioFile == NULL) {
s_audioFile = fopen(WIDGET_SPEAKER_AUDIO_OPUS_FILE_NAME, "wb");
if (s_audioFile == NULL) {
USER_LOG_ERROR("Create tts file error.");
}
if (s_speakerState.state != DJI_WIDGET_SPEAKER_STATE_PLAYING) {
SetSpeakerState(DJI_WIDGET_SPEAKER_STATE_TRANSMITTING);
}
} else if (event == DJI_WIDGET_TRANSMIT_DATA_EVENT_TRANSMIT) {
USER_LOG_INFO("Transmit voice file, offset: %d, size: %d", offset, size);
if (audioFile != NULL) {
fseek(audioFile, offset, SEEK_SET);
writeLen = fwrite(buf, 1, size, audioFile);
if (s_audioFile != NULL) {
fseek(s_audioFile, offset, SEEK_SET);
writeLen = fwrite(buf, 1, size, s_audioFile);
if (writeLen != size) {
USER_LOG_ERROR("Write tts file error %d", writeLen);
}
}
if (s_speakerState.state != DJI_WIDGET_SPEAKER_STATE_PLAYING) {
SetSpeakerState(DJI_WIDGET_SPEAKER_STATE_TRANSMITTING);
}
} else if (event == DJI_WIDGET_TRANSMIT_DATA_EVENT_FINISH) {
USER_LOG_INFO("Close voice file.");
if (audioFile != NULL) {
fclose(audioFile);
if (s_audioFile != NULL) {
fclose(s_audioFile);
}

returnCode = DjiTest_CheckFileMd5Sum(WIDGET_SPEAKER_AUDIO_OPUS_FILE_NAME, buf, size);
Expand All @@ -661,6 +709,9 @@ static T_DjiReturnCode ReceiveAudioData(E_DjiWidgetTransmitDataEvent event,
return DJI_ERROR_SYSTEM_MODULE_CODE_SYSTEM_ERROR;
}

if (s_speakerState.state != DJI_WIDGET_SPEAKER_STATE_PLAYING) {
SetSpeakerState(DJI_WIDGET_SPEAKER_STATE_IDEL);
}
DjiTest_DecodeAudioData();
}

Expand All @@ -686,6 +737,10 @@ static void *DjiTest_WidgetSpeakerTask(void *arg)
if (s_speakerState.state == DJI_WIDGET_SPEAKER_STATE_PLAYING) {
if (s_speakerState.playMode == DJI_WIDGET_SPEAKER_PLAY_MODE_LOOP_PLAYBACK) {
if (s_speakerState.workMode == DJI_WIDGET_SPEAKER_WORK_MODE_VOICE) {
USER_LOG_WARN("Waiting opus decoder finished...");
while (s_isDecodeFinished == false) {
osalHandler->TaskSleepMs(1);
}
djiReturnCode = DjiTest_PlayAudioData();
if (djiReturnCode != DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS) {
USER_LOG_ERROR("Play audio data failed, error: 0x%08llX.", djiReturnCode);
Expand All @@ -696,8 +751,13 @@ static void *DjiTest_WidgetSpeakerTask(void *arg)
USER_LOG_ERROR("Play tts data failed, error: 0x%08llX.", djiReturnCode);
}
}
osalHandler->TaskSleepMs(1000);
} else {
if (s_speakerState.workMode == DJI_WIDGET_SPEAKER_WORK_MODE_VOICE) {
USER_LOG_WARN("Waiting opus decoder finished...");
while (s_isDecodeFinished == false) {
osalHandler->TaskSleepMs(1);
}
djiReturnCode = DjiTest_PlayAudioData();
if (djiReturnCode != DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS) {
USER_LOG_ERROR("Play audio data failed, error: 0x%08llX.", djiReturnCode);
Expand All @@ -714,7 +774,9 @@ static void *DjiTest_WidgetSpeakerTask(void *arg)
USER_LOG_ERROR("lock mutex error: 0x%08llX.", djiReturnCode);
}

s_speakerState.state = DJI_WIDGET_SPEAKER_STATE_IDEL;
if (s_speakerState.playMode == DJI_WIDGET_SPEAKER_PLAY_MODE_SINGLE_PLAY) {
s_speakerState.state = DJI_WIDGET_SPEAKER_STATE_IDEL;
}

djiReturnCode = osalHandler->MutexUnlock(s_speakerMutex);
if (djiReturnCode != DJI_ERROR_SYSTEM_MODULE_CODE_SUCCESS) {
Expand Down

0 comments on commit 3d8e2b5

Please sign in to comment.