mirror of
https://github.com/esphome/esphome.git
synced 2025-01-24 13:24:27 +01:00
Remove afe and try to use algorithm stream
This commit is contained in:
Jesse Hills
parent
0b60a1d9eb
commit
bab9c7c70e
3 changed files with 103 additions and 145 deletions
|
|
@ -7,58 +7,32 @@
|
|||
#include "esphome/core/hal.h"
|
||||
#include "esphome/core/log.h"
|
||||
|
||||
#include <algorithm_stream.h>
|
||||
#include <audio_element.h>
|
||||
#include <audio_hal.h>
|
||||
#include <audio_pipeline.h>
|
||||
#include <filter_resample.h>
|
||||
#include <i2s_stream.h>
|
||||
#include <raw_stream.h>
|
||||
#include <recorder_sr.h>
|
||||
|
||||
#include <board.h>
|
||||
|
||||
namespace esphome {
|
||||
namespace esp_adf {
|
||||
|
||||
static const char *const TAG = "esp_adf.microphone";
|
||||
|
||||
void ESPADFMicrophone::setup() {
|
||||
this->ring_buffer_ = rb_create(BUFFER_SIZE, sizeof(int16_t));
|
||||
this->ring_buffer_ = rb_create(8000, sizeof(int16_t));
|
||||
if (this->ring_buffer_ == nullptr) {
|
||||
ESP_LOGW(TAG, "Could not allocate ring buffer.");
|
||||
this->mark_failed();
|
||||
return;
|
||||
}
|
||||
this->feed_event_queue_ = xQueueCreate(20, sizeof(TaskEvent));
|
||||
this->feed_command_queue_ = xQueueCreate(20, sizeof(CommandEvent));
|
||||
this->fetch_command_queue_ = xQueueCreate(20, sizeof(CommandEvent));
|
||||
|
||||
afe_config_t cfg_afe = {
|
||||
.aec_init = false,
|
||||
.se_init = true,
|
||||
.vad_init = true,
|
||||
.wakenet_init = false,
|
||||
.voice_communication_init = false,
|
||||
.voice_communication_agc_init = false,
|
||||
.voice_communication_agc_gain = 15,
|
||||
.vad_mode = VAD_MODE_3,
|
||||
.wakenet_model_name = nullptr,
|
||||
.wakenet_mode = DET_MODE_2CH_90,
|
||||
.afe_mode = SR_MODE_HIGH_PERF,
|
||||
.afe_perferred_core = 1,
|
||||
.afe_perferred_priority = 5,
|
||||
.afe_ringbuf_size = 50,
|
||||
.memory_alloc_mode = AFE_MEMORY_ALLOC_MORE_PSRAM,
|
||||
.agc_mode = AFE_MN_PEAK_AGC_MODE_3,
|
||||
.pcm_config =
|
||||
{
|
||||
.total_ch_num = 1,
|
||||
.mic_num = 1,
|
||||
.ref_num = 0,
|
||||
.sample_rate = 16000,
|
||||
},
|
||||
.debug_init = false,
|
||||
.debug_hook = {{AFE_DEBUG_HOOK_MASE_TASK_IN, NULL}, {AFE_DEBUG_HOOK_FETCH_TASK_IN, NULL}},
|
||||
};
|
||||
|
||||
this->afe_data_ = this->afe_handle_->create_from_config(&cfg_afe);
|
||||
this->afe_chunk_size_ = this->afe_handle_->get_feed_chunksize(this->afe_data_);
|
||||
this->read_event_queue_ = xQueueCreate(20, sizeof(TaskEvent));
|
||||
this->read_command_queue_ = xQueueCreate(20, sizeof(CommandEvent));
|
||||
}
|
||||
|
||||
void ESPADFMicrophone::start() {
|
||||
|
|
@ -74,42 +48,48 @@ void ESPADFMicrophone::start_() {
|
|||
if (!this->parent_->try_lock()) {
|
||||
return;
|
||||
}
|
||||
this->state_ = microphone::STATE_RUNNING;
|
||||
|
||||
xTaskCreate(ESPADFMicrophone::feed_task, "feed_task", 8192, (void *) this, 0, &this->feed_task_handle_);
|
||||
xTaskCreate(ESPADFMicrophone::fetch_task, "fetch_task", 8192, (void *) this, 0, &this->fetch_task_handle_);
|
||||
xTaskCreate(ESPADFMicrophone::read_task, "read_task", 8192, (void *) this, 0, &this->read_task_handle_);
|
||||
}
|
||||
|
||||
void ESPADFMicrophone::feed_task(void *params) {
|
||||
void ESPADFMicrophone::read_task(void *params) {
|
||||
ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params;
|
||||
TaskEvent event;
|
||||
|
||||
event.type = TaskEventType::STARTING;
|
||||
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
|
||||
|
||||
size_t buffer_size = this_mic->afe_chunk_size_ * sizeof(int16_t);
|
||||
|
||||
ExternalRAMAllocator<int16_t> allocator(ExternalRAMAllocator<int16_t>::ALLOW_FAILURE);
|
||||
int16_t *afe_buffer = allocator.allocate(this_mic->afe_chunk_size_);
|
||||
if (afe_buffer == nullptr) {
|
||||
int16_t *buffer = allocator.allocate(BUFFER_SIZE / sizeof(int16_t));
|
||||
if (buffer == nullptr) {
|
||||
event.type = TaskEventType::WARNING;
|
||||
event.err = ESP_ERR_NO_MEM;
|
||||
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
|
||||
|
||||
event.type = TaskEventType::STOPPED;
|
||||
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
|
||||
event.err = ESP_OK;
|
||||
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
|
||||
|
||||
while (true) {
|
||||
delay(10);
|
||||
};
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
event.type = TaskEventType::STARTING;
|
||||
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
|
||||
|
||||
audio_pipeline_cfg_t pipeline_cfg = {
|
||||
.rb_size = 8 * 1024,
|
||||
};
|
||||
audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg);
|
||||
|
||||
i2s_driver_config_t i2s_config = {
|
||||
.mode = (i2s_mode_t) (I2S_MODE_MASTER | I2S_MODE_RX),
|
||||
.sample_rate = 44100,
|
||||
.sample_rate = 16000,
|
||||
.bits_per_sample = I2S_BITS_PER_SAMPLE_16BIT,
|
||||
.channel_format = I2S_CHANNEL_FMT_RIGHT_LEFT,
|
||||
.communication_format = I2S_COMM_FORMAT_STAND_I2S,
|
||||
.intr_alloc_flags = ESP_INTR_FLAG_LEVEL2 | ESP_INTR_FLAG_IRAM,
|
||||
.dma_buf_count = 3,
|
||||
.dma_buf_len = 300,
|
||||
.dma_buf_count = 8,
|
||||
.dma_buf_len = 128,
|
||||
.use_apll = false,
|
||||
.tx_desc_auto_clear = true,
|
||||
.fixed_mclk = 0,
|
||||
|
|
@ -117,11 +97,6 @@ void ESPADFMicrophone::feed_task(void *params) {
|
|||
.bits_per_chan = I2S_BITS_PER_CHAN_DEFAULT,
|
||||
};
|
||||
|
||||
audio_pipeline_cfg_t pipeline_cfg = {
|
||||
.rb_size = 8 * 1024,
|
||||
};
|
||||
audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg);
|
||||
|
||||
i2s_stream_cfg_t i2s_cfg = {
|
||||
.type = AUDIO_STREAM_READER,
|
||||
.i2s_config = i2s_config,
|
||||
|
|
@ -141,7 +116,7 @@ void ESPADFMicrophone::feed_task(void *params) {
|
|||
audio_element_handle_t i2s_stream_reader = i2s_stream_init(&i2s_cfg);
|
||||
|
||||
rsp_filter_cfg_t rsp_cfg = {
|
||||
.src_rate = 44100,
|
||||
.src_rate = 16000,
|
||||
.src_ch = 2,
|
||||
.dest_rate = 16000,
|
||||
.dest_bits = 16,
|
||||
|
|
@ -162,6 +137,30 @@ void ESPADFMicrophone::feed_task(void *params) {
|
|||
};
|
||||
audio_element_handle_t filter = rsp_filter_init(&rsp_cfg);
|
||||
|
||||
algorithm_stream_cfg_t algo_cfg = {
|
||||
.input_type = ALGORITHM_STREAM_INPUT_TYPE1,
|
||||
.task_stack = 10 * 1024,
|
||||
.task_prio = ALGORITHM_STREAM_TASK_PERIOD,
|
||||
.task_core = ALGORITHM_STREAM_PINNED_TO_CORE,
|
||||
.out_rb_size = ALGORITHM_STREAM_RINGBUFFER_SIZE,
|
||||
.stack_in_ext = true,
|
||||
.rec_linear_factor = 1,
|
||||
.ref_linear_factor = 1,
|
||||
.debug_input = false,
|
||||
.swap_ch = false,
|
||||
// .algo_mask = ALGORITHM_STREAM_USE_AGC,
|
||||
// .algo_mask = (ALGORITHM_STREAM_USE_AEC | ALGORITHM_STREAM_USE_AGC | ALGORITHM_STREAM_USE_NS),
|
||||
// .algo_mask = (ALGORITHM_STREAM_USE_AGC | ALGORITHM_STREAM_USE_NS),
|
||||
.algo_mask = (ALGORITHM_STREAM_USE_AEC | ALGORITHM_STREAM_USE_NS),
|
||||
// .algo_mask = (ALGORITHM_STREAM_USE_NS),
|
||||
.sample_rate = 16000,
|
||||
.mic_ch = 1,
|
||||
.agc_gain = 10,
|
||||
.aec_low_cost = false,
|
||||
};
|
||||
|
||||
// audio_element_handle_t algo_stream = algo_stream_init(&algo_cfg);
|
||||
|
||||
raw_stream_cfg_t raw_cfg = {
|
||||
.type = AUDIO_STREAM_READER,
|
||||
.out_rb_size = 8 * 1024,
|
||||
|
|
@ -170,28 +169,33 @@ void ESPADFMicrophone::feed_task(void *params) {
|
|||
|
||||
audio_pipeline_register(pipeline, i2s_stream_reader, "i2s");
|
||||
audio_pipeline_register(pipeline, filter, "filter");
|
||||
// audio_pipeline_register(pipeline, algo_stream, "algo");
|
||||
audio_pipeline_register(pipeline, raw_read, "raw");
|
||||
|
||||
const char *link_tag[3] = {"i2s", "filter", "raw"};
|
||||
const char *link_tag[4] = {
|
||||
"i2s",
|
||||
"filter",
|
||||
// "algo",
|
||||
"raw",
|
||||
};
|
||||
audio_pipeline_link(pipeline, &link_tag[0], 3);
|
||||
|
||||
audio_pipeline_run(pipeline);
|
||||
|
||||
event.type = TaskEventType::STARTED;
|
||||
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
|
||||
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
|
||||
|
||||
CommandEvent command_event;
|
||||
size_t fill_count = 0;
|
||||
|
||||
while (true) {
|
||||
if (xQueueReceive(this_mic->feed_command_queue_, &command_event, 0) == pdTRUE) {
|
||||
if (xQueueReceive(this_mic->read_command_queue_, &command_event, 0) == pdTRUE) {
|
||||
if (command_event.stop) {
|
||||
// Stop signal from main thread
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int bytes_read = raw_stream_read(raw_read, (char *) (afe_buffer + fill_count), buffer_size - fill_count);
|
||||
int bytes_read = raw_stream_read(raw_read, (char *) buffer, BUFFER_SIZE);
|
||||
|
||||
if (bytes_read == -2 || bytes_read == 0) {
|
||||
// No data in buffers to read.
|
||||
|
|
@ -199,72 +203,43 @@ void ESPADFMicrophone::feed_task(void *params) {
|
|||
} else if (bytes_read < 0) {
|
||||
event.type = TaskEventType::WARNING;
|
||||
event.err = bytes_read;
|
||||
xQueueSend(this_mic->feed_event_queue_, &event, 0);
|
||||
xQueueSend(this_mic->read_event_queue_, &event, 0);
|
||||
continue;
|
||||
}
|
||||
|
||||
event.type = TaskEventType::RUNNING;
|
||||
event.err = ESP_OK;
|
||||
xQueueSend(this_mic->feed_event_queue_, &event, 0);
|
||||
event.err = bytes_read;
|
||||
xQueueSend(this_mic->read_event_queue_, &event, 0);
|
||||
|
||||
fill_count += bytes_read;
|
||||
|
||||
if (fill_count == buffer_size) {
|
||||
this_mic->afe_handle_->feed(this_mic->afe_data_, afe_buffer);
|
||||
fill_count -= buffer_size;
|
||||
int available = rb_bytes_available(this_mic->ring_buffer_);
|
||||
if (available < bytes_read) {
|
||||
rb_read(this_mic->ring_buffer_, nullptr, bytes_read - available, 0);
|
||||
}
|
||||
rb_write(this_mic->ring_buffer_, (char *) buffer, bytes_read, 0);
|
||||
}
|
||||
|
||||
allocator.deallocate(afe_buffer, this_mic->afe_chunk_size_);
|
||||
allocator.deallocate(buffer, BUFFER_SIZE / sizeof(int16_t));
|
||||
|
||||
audio_pipeline_stop(pipeline);
|
||||
audio_pipeline_wait_for_stop(pipeline);
|
||||
audio_pipeline_terminate(pipeline);
|
||||
|
||||
event.type = TaskEventType::STOPPING;
|
||||
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
|
||||
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
|
||||
|
||||
audio_pipeline_unregister(pipeline, i2s_stream_reader);
|
||||
audio_pipeline_unregister(pipeline, filter);
|
||||
// audio_pipeline_unregister(pipeline, algo_stream);
|
||||
audio_pipeline_unregister(pipeline, raw_read);
|
||||
|
||||
audio_pipeline_deinit(pipeline);
|
||||
audio_element_deinit(i2s_stream_reader);
|
||||
audio_element_deinit(filter);
|
||||
// audio_element_deinit(algo_stream);
|
||||
audio_element_deinit(raw_read);
|
||||
|
||||
event.type = TaskEventType::STOPPED;
|
||||
xQueueSend(this_mic->feed_event_queue_, &event, portMAX_DELAY);
|
||||
|
||||
while (true) {
|
||||
delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
void ESPADFMicrophone::fetch_task(void *params) {
|
||||
ESPADFMicrophone *this_mic = (ESPADFMicrophone *) params;
|
||||
|
||||
CommandEvent event;
|
||||
|
||||
while (true) {
|
||||
if (xQueueReceive(this_mic->fetch_command_queue_, &event, 0) == pdTRUE) {
|
||||
if (event.stop) {
|
||||
// Stop signal from main thread
|
||||
break;
|
||||
}
|
||||
}
|
||||
afe_fetch_result_t *result = this_mic->afe_handle_->fetch(this_mic->afe_data_);
|
||||
|
||||
if (result == nullptr) {
|
||||
continue;
|
||||
}
|
||||
|
||||
int available = rb_bytes_available(this_mic->ring_buffer_);
|
||||
if (available < result->data_size) {
|
||||
rb_read(this_mic->ring_buffer_, nullptr, result->data_size - available, 0);
|
||||
}
|
||||
rb_write(this_mic->ring_buffer_, (char *) result->data, result->data_size, 0);
|
||||
}
|
||||
xQueueSend(this_mic->read_event_queue_, &event, portMAX_DELAY);
|
||||
|
||||
while (true) {
|
||||
delay(10);
|
||||
|
|
@ -272,31 +247,26 @@ void ESPADFMicrophone::fetch_task(void *params) {
|
|||
}
|
||||
|
||||
void ESPADFMicrophone::stop() {
|
||||
if (this->state_ == microphone::STATE_STOPPED || this->is_failed())
|
||||
if (this->state_ == microphone::STATE_STOPPED || this->state_ == microphone::STATE_STOPPING || this->is_failed())
|
||||
return;
|
||||
if (this->state_ == microphone::STATE_STARTING) {
|
||||
this->state_ = microphone::STATE_STOPPED;
|
||||
return;
|
||||
}
|
||||
this->state_ = microphone::STATE_STOPPING;
|
||||
}
|
||||
|
||||
void ESPADFMicrophone::stop_() {
|
||||
ESP_LOGD(TAG, "Stopping microphone");
|
||||
CommandEvent command_event;
|
||||
command_event.stop = true;
|
||||
xQueueSendToFront(this->feed_command_queue_, &command_event, portMAX_DELAY);
|
||||
xQueueSendToFront(this->fetch_command_queue_, &command_event, portMAX_DELAY);
|
||||
xQueueSendToFront(this->read_command_queue_, &command_event, portMAX_DELAY);
|
||||
ESP_LOGD(TAG, "Stopping microphone");
|
||||
}
|
||||
|
||||
size_t ESPADFMicrophone::read(int16_t *buf, size_t len) {
|
||||
if (rb_bytes_available(this->ring_buffer_) == 0) {
|
||||
return 0; // No data
|
||||
}
|
||||
int bytes_read = rb_read(this->ring_buffer_, (char *) buf, len, 0);
|
||||
|
||||
if (bytes_read == -2 || bytes_read == 0) {
|
||||
if (bytes_read == -4 || bytes_read == -2 || bytes_read == 0) {
|
||||
// No data in buffers to read.
|
||||
return 0;
|
||||
} else if (bytes_read < 0) {
|
||||
ESP_LOGW(TAG, "Error reading from I2S microphone");
|
||||
ESP_LOGW(TAG, "Error reading from I2S microphone %s (%d)", esp_err_to_name(bytes_read), bytes_read);
|
||||
this->status_set_warning();
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -315,22 +285,24 @@ void ESPADFMicrophone::read_() {
|
|||
|
||||
void ESPADFMicrophone::watch_() {
|
||||
TaskEvent event;
|
||||
if (xQueueReceive(this->feed_event_queue_, &event, 0) == pdTRUE) {
|
||||
if (xQueueReceive(this->read_event_queue_, &event, 0) == pdTRUE) {
|
||||
switch (event.type) {
|
||||
case TaskEventType::STARTING:
|
||||
case TaskEventType::STARTED:
|
||||
case TaskEventType::STOPPING:
|
||||
break;
|
||||
case TaskEventType::STARTED:
|
||||
ESP_LOGD(TAG, "Microphone started");
|
||||
this->state_ = microphone::STATE_RUNNING;
|
||||
break;
|
||||
case TaskEventType::RUNNING:
|
||||
this->status_clear_warning();
|
||||
// ESP_LOGD(TAG, "Putting %d bytes into ring buffer", event.err);
|
||||
break;
|
||||
case TaskEventType::STOPPED:
|
||||
this->parent_->unlock();
|
||||
this->state_ = microphone::STATE_STOPPED;
|
||||
vTaskDelete(this->feed_task_handle_);
|
||||
vTaskDelete(this->fetch_task_handle_);
|
||||
this->feed_task_handle_ = nullptr;
|
||||
this->fetch_task_handle_ = nullptr;
|
||||
vTaskDelete(this->read_task_handle_);
|
||||
this->read_task_handle_ = nullptr;
|
||||
ESP_LOGD(TAG, "Microphone stopped");
|
||||
break;
|
||||
case TaskEventType::WARNING:
|
||||
|
|
@ -342,8 +314,10 @@ void ESPADFMicrophone::watch_() {
|
|||
}
|
||||
|
||||
void ESPADFMicrophone::loop() {
|
||||
this->watch_();
|
||||
switch (this->state_) {
|
||||
case microphone::STATE_STOPPED:
|
||||
case microphone::STATE_STOPPING:
|
||||
break;
|
||||
case microphone::STATE_STARTING:
|
||||
this->start_();
|
||||
|
|
@ -353,9 +327,6 @@ void ESPADFMicrophone::loop() {
|
|||
this->read_();
|
||||
}
|
||||
break;
|
||||
case microphone::STATE_STOPPING:
|
||||
this->stop_();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,10 +7,6 @@
|
|||
#include "esphome/components/microphone/microphone.h"
|
||||
#include "esphome/core/component.h"
|
||||
|
||||
#include <audio_element.h>
|
||||
#include <audio_pipeline.h>
|
||||
#include <esp_afe_sr_iface.h>
|
||||
#include <esp_afe_sr_models.h>
|
||||
#include <ringbuf.h>
|
||||
|
||||
namespace esphome {
|
||||
|
|
@ -28,25 +24,16 @@ class ESPADFMicrophone : public ESPADFPipeline, public microphone::Microphone, p
|
|||
|
||||
protected:
|
||||
void start_();
|
||||
void stop_();
|
||||
void read_();
|
||||
void watch_();
|
||||
|
||||
static void feed_task(void *params);
|
||||
static void fetch_task(void *params);
|
||||
|
||||
const esp_afe_sr_iface_t *afe_handle_{&ESP_AFE_SR_HANDLE};
|
||||
esp_afe_sr_data_t *afe_data_{nullptr};
|
||||
size_t afe_chunk_size_{0};
|
||||
static void read_task(void *params);
|
||||
|
||||
ringbuf_handle_t ring_buffer_;
|
||||
|
||||
TaskHandle_t feed_task_handle_{nullptr};
|
||||
QueueHandle_t feed_event_queue_;
|
||||
QueueHandle_t feed_command_queue_;
|
||||
|
||||
TaskHandle_t fetch_task_handle_{nullptr};
|
||||
QueueHandle_t fetch_command_queue_;
|
||||
TaskHandle_t read_task_handle_{nullptr};
|
||||
QueueHandle_t read_event_queue_;
|
||||
QueueHandle_t read_command_queue_;
|
||||
};
|
||||
|
||||
} // namespace esp_adf
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ void ESPADFSpeaker::start_() {
|
|||
if (!this->parent_->try_lock()) {
|
||||
return; // Waiting for another i2s component to return lock
|
||||
}
|
||||
this->state_ = speaker::STATE_RUNNING;
|
||||
|
||||
xTaskCreate(ESPADFSpeaker::player_task, "speaker_task", 8192, (void *) this, 0, &this->player_task_handle_);
|
||||
}
|
||||
|
|
@ -211,9 +210,11 @@ void ESPADFSpeaker::watch_() {
|
|||
if (xQueueReceive(this->event_queue_, &event, 0) == pdTRUE) {
|
||||
switch (event.type) {
|
||||
case TaskEventType::STARTING:
|
||||
case TaskEventType::STARTED:
|
||||
case TaskEventType::STOPPING:
|
||||
break;
|
||||
case TaskEventType::STARTED:
|
||||
this->state_ = speaker::STATE_RUNNING;
|
||||
break;
|
||||
case TaskEventType::RUNNING:
|
||||
this->status_clear_warning();
|
||||
break;
|
||||
|
|
@ -232,13 +233,12 @@ void ESPADFSpeaker::watch_() {
|
|||
}
|
||||
|
||||
void ESPADFSpeaker::loop() {
|
||||
this->watch_();
|
||||
switch (this->state_) {
|
||||
case speaker::STATE_STARTING:
|
||||
this->start_();
|
||||
break;
|
||||
case speaker::STATE_RUNNING:
|
||||
this->watch_();
|
||||
break;
|
||||
case speaker::STATE_STOPPING:
|
||||
case speaker::STATE_STOPPED:
|
||||
break;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue