From fa02d10221faf3a76060ca3a48b13080c4de6ea0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:15:45 +0000 Subject: [PATCH] Add t2bce driver stack --- drivers/staging/t2bce/Kconfig | 40 + drivers/staging/t2bce/Makefile | 5 + drivers/staging/t2bce/t2bce_audio/Makefile | 7 + drivers/staging/t2bce/t2bce_audio/audio.c | 929 ++++++++++++ drivers/staging/t2bce/t2bce_audio/audio.h | 148 ++ .../staging/t2bce/t2bce_audio/description.h | 42 + drivers/staging/t2bce/t2bce_audio/pcm.c | 520 +++++++ drivers/staging/t2bce/t2bce_audio/pcm.h | 17 + drivers/staging/t2bce/t2bce_audio/protocol.c | 347 +++++ drivers/staging/t2bce/t2bce_audio/protocol.h | 147 ++ .../staging/t2bce/t2bce_audio/protocol_bce.c | 260 ++++ .../staging/t2bce/t2bce_audio/protocol_bce.h | 72 + drivers/staging/t2bce/t2bce_core/Makefile | 7 + .../t2bce_core/include/t2bce_core_transport.h | 83 ++ drivers/staging/t2bce/t2bce_core/mailbox.c | 234 +++ drivers/staging/t2bce/t2bce_core/mailbox.h | 64 + drivers/staging/t2bce/t2bce_core/t2bce.h | 51 + drivers/staging/t2bce/t2bce_core/t2bce_main.c | 766 ++++++++++ drivers/staging/t2bce/t2bce_core/transport.c | 408 ++++++ drivers/staging/t2bce/t2bce_dma/Makefile | 6 + .../t2bce/t2bce_dma/include/t2bce_dma_queue.h | 182 +++ drivers/staging/t2bce/t2bce_dma/queue.c | 625 ++++++++ drivers/staging/t2bce/t2bce_vhci/Makefile | 7 + drivers/staging/t2bce/t2bce_vhci/command.h | 206 +++ drivers/staging/t2bce/t2bce_vhci/queue.c | 269 ++++ drivers/staging/t2bce/t2bce_vhci/queue.h | 76 + drivers/staging/t2bce/t2bce_vhci/transfer.c | 1261 ++++++++++++++++ drivers/staging/t2bce/t2bce_vhci/transfer.h | 86 ++ drivers/staging/t2bce/t2bce_vhci/vhci.c | 1293 +++++++++++++++++ drivers/staging/t2bce/t2bce_vhci/vhci.h | 69 + 30 files changed, 8227 insertions(+) create mode 100644 drivers/staging/t2bce/Kconfig create mode 100644 drivers/staging/t2bce/Makefile create mode 100644 drivers/staging/t2bce/t2bce_audio/Makefile create mode 100644 drivers/staging/t2bce/t2bce_audio/audio.c create mode 100644 drivers/staging/t2bce/t2bce_audio/audio.h create mode 100644 drivers/staging/t2bce/t2bce_audio/description.h create mode 100644 drivers/staging/t2bce/t2bce_audio/pcm.c create mode 100644 drivers/staging/t2bce/t2bce_audio/pcm.h create mode 100644 drivers/staging/t2bce/t2bce_audio/protocol.c create mode 100644 drivers/staging/t2bce/t2bce_audio/protocol.h create mode 100644 drivers/staging/t2bce/t2bce_audio/protocol_bce.c create mode 100644 drivers/staging/t2bce/t2bce_audio/protocol_bce.h create mode 100644 drivers/staging/t2bce/t2bce_core/Makefile create mode 100644 drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h create mode 100644 drivers/staging/t2bce/t2bce_core/mailbox.c create mode 100644 drivers/staging/t2bce/t2bce_core/mailbox.h create mode 100644 drivers/staging/t2bce/t2bce_core/t2bce.h create mode 100644 drivers/staging/t2bce/t2bce_core/t2bce_main.c create mode 100644 drivers/staging/t2bce/t2bce_core/transport.c create mode 100644 drivers/staging/t2bce/t2bce_dma/Makefile create mode 100644 drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h create mode 100644 drivers/staging/t2bce/t2bce_dma/queue.c create mode 100644 drivers/staging/t2bce/t2bce_vhci/Makefile create mode 100644 drivers/staging/t2bce/t2bce_vhci/command.h create mode 100644 drivers/staging/t2bce/t2bce_vhci/queue.c create mode 100644 drivers/staging/t2bce/t2bce_vhci/queue.h create mode 100644 drivers/staging/t2bce/t2bce_vhci/transfer.c create mode 100644 drivers/staging/t2bce/t2bce_vhci/transfer.h create mode 100644 drivers/staging/t2bce/t2bce_vhci/vhci.c create mode 100644 drivers/staging/t2bce/t2bce_vhci/vhci.h diff --git a/drivers/staging/t2bce/Kconfig b/drivers/staging/t2bce/Kconfig new file mode 100644 index 000000000..112bc3435 --- /dev/null +++ b/drivers/staging/t2bce/Kconfig @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "Apple T2 Buffer Copy Engine support" + +config T2BCE_DMA + tristate + help + DMA queue support for the Apple T2 Buffer Copy Engine. + +config T2BCE_CORE + tristate "Apple T2 Buffer Copy Engine core" + depends on X86_64 && PCI + select T2BCE_DMA + help + Core transport driver for the Buffer Copy Engine in Apple T2 Macs. + It provides the mailbox, DMA queue coordination, power management, + and transport API used by the T2 VHCI and audio drivers. + + If compiled as a module, it will be called t2bce_core. + +config T2BCE_VHCI + tristate "Apple T2 virtual USB host controller" + depends on T2BCE_CORE && USB + help + Virtual USB 2.0 host controller for devices exposed by bridgeOS, + including the internal keyboard, trackpad, Touch Bar, and camera. + + If compiled as a module, it will be called t2bce_vhci. + +config T2BCE_AUDIO + tristate "Apple T2 audio support" + depends on T2BCE_CORE && SND + select SND_PCM + select SND_JACK + help + ALSA driver for the audio endpoints exposed by bridgeOS on Apple + T2 Macs. + + If compiled as a module, it will be called t2bce_audio. + +endmenu diff --git a/drivers/staging/t2bce/Makefile b/drivers/staging/t2bce/Makefile new file mode 100644 index 000000000..91eebdc8d --- /dev/null +++ b/drivers/staging/t2bce/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_DMA) += t2bce_dma/ +obj-$(CONFIG_T2BCE_CORE) += t2bce_core/ +obj-$(CONFIG_T2BCE_VHCI) += t2bce_vhci/ +obj-$(CONFIG_T2BCE_AUDIO) += t2bce_audio/ diff --git a/drivers/staging/t2bce/t2bce_audio/Makefile b/drivers/staging/t2bce/t2bce_audio/Makefile new file mode 100644 index 000000000..1fba36e10 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_AUDIO) += t2bce_audio.o +t2bce_audio-y := audio.o protocol.o protocol_bce.o pcm.o + +ccflags-y += -DWITHOUT_NVME_PATCH +ccflags-y += -I$(src) +ccflags-y += -I$(src)/../t2bce_core/include diff --git a/drivers/staging/t2bce/t2bce_audio/audio.c b/drivers/staging/t2bce/t2bce_audio/audio.c new file mode 100644 index 000000000..60f552c06 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/audio.c @@ -0,0 +1,929 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "audio.h" +#include "pcm.h" +#include + +static int t2audio_alsa_index = SNDRV_DEFAULT_IDX1; +static char *t2audio_alsa_id = SNDRV_DEFAULT_STR1; + +static dev_t t2audio_chrdev; +static struct class *t2audio_class; + +static int t2audio_init_cmd(struct t2audio_device *a); +static int t2audio_init_bs(struct t2audio_device *a); +static void t2audio_init_dev(struct t2audio_device *a, t2audio_device_id_t dev_id); +static void t2audio_free_dev(struct t2audio_subdevice *sdev); +static void t2audio_reset_stream(struct t2audio_stream *stream); +static void t2audio_reset_streams(struct t2audio_device *a); +static void t2audio_reset_clock(struct t2audio_device *a); +static void t2audio_resume_work(struct work_struct *ws); +static void t2audio_resume_complete(void *userdata); +static void t2audio_pm_prepare_client(void *userdata); +static void t2audio_pm_shutdown_client(void *userdata); + +static const struct t2bce_core_client_pm_ops t2audio_pm_ops = { + .shutdown = t2audio_pm_shutdown_client, + .pm_prepare = t2audio_pm_prepare_client, +}; + +static int t2audio_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + struct t2audio_device *t2audio = NULL; + struct t2audio_subdevice *sdev = NULL; + int status = 0; + u32 cfg; + + pr_debug("t2bce_audio: capturing our device\n"); + + if (pci_enable_device(dev)) + return -ENODEV; + if (pci_request_regions(dev, "t2bce_audio")) { + status = -ENODEV; + goto fail; + } + pci_set_master(dev); + + t2audio = kzalloc(sizeof(struct t2audio_device), GFP_KERNEL); + if (!t2audio) { + status = -ENOMEM; + goto fail; + } + + t2audio->bce = t2bce_core_client_get(&dev->dev); + if (IS_ERR(t2audio->bce)) { + status = PTR_ERR(t2audio->bce); + t2audio->bce = NULL; + if (status != -EPROBE_DEFER) + dev_warn(&dev->dev, "t2bce_audio: Failed to get BCE client: %d\n", status); + goto fail; + } + + t2audio->pci = dev; + pci_set_drvdata(dev, t2audio); + t2bce_core_client_set_resume_complete_callback(t2audio->bce, t2audio_resume_complete, t2audio); + t2bce_core_client_set_pm_ops(t2audio->bce, &t2audio_pm_ops, t2audio); + + t2audio->devt = t2audio_chrdev; + t2audio->dev = device_create(t2audio_class, &dev->dev, t2audio->devt, NULL, "t2bce_audio"); + if (IS_ERR_OR_NULL(t2audio->dev)) { + status = PTR_ERR(t2audio->dev); + goto fail; + } + init_completion(&t2audio->remote_alive); + INIT_WORK(&t2audio->resume_work, t2audio_resume_work); + INIT_LIST_HEAD(&t2audio->subdevice_list); + spin_lock_init(&t2audio->clock_lock); + t2audio->clock_samples = 0; + t2audio->clock_offset_valid = false; + + /* Init: set an unknown flag in the bitset */ + if (pci_read_config_dword(dev, 4, &cfg)) + dev_warn(&dev->dev, "t2bce_audio: pci_read_config_dword fail\n"); + if (pci_write_config_dword(dev, 4, cfg | 6u)) + dev_warn(&dev->dev, "t2bce_audio: pci_write_config_dword fail\n"); + + pr_debug("t2bce_audio: bs len = %llx\n", pci_resource_len(dev, 0)); + t2audio->reg_mem_bs_dma = pci_resource_start(dev, 0); + t2audio->reg_mem_bs = pci_iomap(dev, 0, 0); + t2audio->reg_mem_cfg = pci_iomap(dev, 4, 0); + + t2audio->reg_mem_gpr = (u32 __iomem *) ((u8 __iomem *) t2audio->reg_mem_cfg + 0xC000); + + if (IS_ERR_OR_NULL(t2audio->reg_mem_bs) || IS_ERR_OR_NULL(t2audio->reg_mem_cfg)) { + dev_warn(&dev->dev, "t2bce_audio: Failed to pci_iomap required regions\n"); + goto fail; + } + + if (t2audio_bce_init(t2audio)) { + dev_warn(&dev->dev, "t2bce_audio: Failed to init BCE command transport\n"); + goto fail; + } + + if (snd_card_new(t2audio->dev, t2audio_alsa_index, t2audio_alsa_id, THIS_MODULE, 0, &t2audio->card)) { + dev_err(&dev->dev, "t2bce_audio: Failed to create ALSA card\n"); + goto fail; + } + + strcpy(t2audio->card->shortname, "Apple T2 Audio"); + strcpy(t2audio->card->longname, "Apple T2 Audio"); + strcpy(t2audio->card->mixername, "Apple T2 Audio"); + /* Dynamic alsa ids start at 100 */ + t2audio->next_alsa_id = 100; + + if (t2audio_init_cmd(t2audio)) { + dev_err(&dev->dev, "t2bce_audio: Failed to initialize over BCE\n"); + goto fail_snd; + } + + if (t2audio_init_bs(t2audio)) { + dev_err(&dev->dev, "t2bce_audio: Failed to initialize BufferStruct\n"); + goto fail_snd; + } + + if ((status = t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_ON))) { + dev_err(&dev->dev, "Failed to set remote access\n"); + goto fail_snd; + } + + if (snd_card_register(t2audio->card)) { + dev_err(&dev->dev, "t2bce_audio: Failed to register ALSA sound device\n"); + goto fail_snd; + } + + list_for_each_entry(sdev, &t2audio->subdevice_list, list) { + struct t2audio_buffer_struct_device *dev = &t2audio->bs->devices[sdev->buf_id]; + + if (sdev->out_stream_cnt == 1 && !strcmp(dev->name, "Speaker")) { + struct snd_pcm_hardware *hw = sdev->out_streams[0].alsa_hw_desc; + + snprintf(t2audio->card->driver, sizeof(t2audio->card->driver) / sizeof(char), "AppleT2x%d", hw->channels_min); + } + } + + return 0; + +fail_snd: + snd_card_free(t2audio->card); +fail: + if (t2audio) { + if (t2audio->dev) + device_destroy(t2audio_class, t2audio->devt); + if (!IS_ERR_OR_NULL(t2audio->reg_mem_bs)) + pci_iounmap(dev, t2audio->reg_mem_bs); + if (!IS_ERR_OR_NULL(t2audio->reg_mem_cfg)) + pci_iounmap(dev, t2audio->reg_mem_cfg); + if (t2audio->bce) + t2bce_core_client_set_pm_ops(t2audio->bce, NULL, NULL); + t2bce_core_client_put(t2audio->bce); + kfree(t2audio); + } + + pci_release_regions(dev); + pci_disable_device(dev); + + if (!status) + status = -EINVAL; + return status; +} + + + +static void t2audio_remove(struct pci_dev *dev) +{ + struct t2audio_subdevice *sdev; + struct t2audio_device *t2audio = pci_get_drvdata(dev); + + cancel_work_sync(&t2audio->resume_work); + snd_card_free(t2audio->card); + while (!list_empty(&t2audio->subdevice_list)) { + sdev = list_first_entry(&t2audio->subdevice_list, struct t2audio_subdevice, list); + list_del(&sdev->list); + t2audio_free_dev(sdev); + } + pci_iounmap(dev, t2audio->reg_mem_bs); + pci_iounmap(dev, t2audio->reg_mem_cfg); + device_destroy(t2audio_class, t2audio->devt); + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + t2bce_core_client_set_pm_ops(t2audio->bce, NULL, NULL); + t2bce_core_client_put(t2audio->bce); + kfree(t2audio); +} + +static int t2audio_quiesce(struct t2audio_device *t2audio, bool suspend_pcm) +{ + struct t2audio_subdevice *sdev; + size_t i; + int status; + + if (t2audio->pm_quiesced) + return 0; + + cancel_work_sync(&t2audio->resume_work); + t2audio->resume_deferred = false; + + /* Suspend PCM streams */ + list_for_each_entry(sdev, &t2audio->subdevice_list, list) { + bool stopped_io = false; + + for (i = 0; i < sdev->out_stream_cnt; i++) { + if (!smp_load_acquire(&sdev->out_streams[i].started)) + continue; + stopped_io = true; + smp_store_release(&sdev->out_streams[i].started, 0); + } + + for (i = 0; i < sdev->in_stream_cnt; i++) { + if (!smp_load_acquire(&sdev->in_streams[i].started)) + continue; + stopped_io = true; + smp_store_release(&sdev->in_streams[i].started, 0); + } + + if (stopped_io) + t2audio_cmd_stop_io(sdev->a, sdev->dev_id); + + if (suspend_pcm && sdev->pcm) + snd_pcm_suspend_all(sdev->pcm); + } + + status = t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_OFF); + if (status) + dev_warn(t2audio->dev, "Failed to reset remote access\n"); + else + t2audio->pm_quiesced = true; + + return status; +} + +static int t2audio_suspend(struct device *dev) +{ + struct t2audio_device *t2audio = pci_get_drvdata(to_pci_dev(dev)); + int status; + + pr_debug("t2bce_audio: suspend entry\n"); + status = t2audio_quiesce(t2audio, true); + pci_disable_device(t2audio->pci); + pr_info("t2bce_audio: suspend exit status=%d\n", status); + return 0; +} + +static void t2audio_pm_prepare_client(void *userdata) +{ + t2audio_quiesce(userdata, true); +} + +static void t2audio_pm_shutdown_client(void *userdata) +{ + t2audio_quiesce(userdata, false); +} + +static void t2audio_shutdown(struct pci_dev *dev) +{ + struct t2audio_device *t2audio = pci_get_drvdata(dev); + + if (!t2audio) + return; + + /* + * Shutdown is not remove: keep allocations intact, but stop active audio + * IO and revoke remote access while the BCE command transport is still + * alive. The t2bce shutdown path will close the shared bus afterwards. + */ + t2audio_quiesce(t2audio, false); + pci_disable_device(t2audio->pci); +} + +static int t2audio_resume(struct device *dev) +{ + int status; + struct t2audio_device *t2audio = pci_get_drvdata(to_pci_dev(dev)); + bool no_state_resume = t2bce_core_client_no_state_resume(t2audio->bce); + const char *path = no_state_resume ? "no-state" : "stateful"; + + if ((status = pci_enable_device(t2audio->pci))) { + pr_info("t2bce_audio: resume exit status=%d path=%s\n", status, path); + return status; + } + pci_set_master(t2audio->pci); + + /* we are deferring t2audio resume here until vhci is finished*/ + if (!no_state_resume) { + t2audio->resume_deferred = true; + return 0; + } + + if ((status = t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_ON))) { + dev_err(t2audio->dev, "Failed to set remote access\n"); + pr_info("t2bce_audio: resume exit status=%d path=%s\n", status, path); + return status; + } + + t2audio->resume_deferred = false; + t2audio->pm_quiesced = false; + t2audio_reset_clock(t2audio); + t2audio_reset_streams(t2audio); + + pr_info("t2bce_audio: resume exit status=0 path=%s\n", path); + return 0; +} + +static void t2audio_resume_work(struct work_struct *ws) +{ + struct t2audio_device *t2audio = container_of(ws, struct t2audio_device, resume_work); + + if (!t2audio->resume_deferred) + return; + + if (t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_ON)) { + t2audio->resume_deferred = false; + dev_err(t2audio->dev, "Deferred remote access enable failed\n"); + pr_info("t2bce_audio: resume deferred path failed\n"); + return; + } + + t2audio->resume_deferred = false; + t2audio->pm_quiesced = false; + t2audio_reset_clock(t2audio); + t2audio_reset_streams(t2audio); + pr_info("t2bce_audio: resume deferred path complete\n"); +} + +static void t2audio_resume_complete(void *userdata) +{ + struct t2audio_device *t2audio = userdata; + + if (!t2audio || !t2audio->resume_deferred) + return; + + schedule_work(&t2audio->resume_work); +} + +static void t2audio_reset_stream(struct t2audio_stream *stream) +{ + smp_store_release(&stream->started, 0); + stream->waiting_for_first_ts = true; + stream->remote_timestamp = 0; + stream->timestamp_accept_after = 0; + stream->clock_offset_ns = 0; + stream->timestamp_seed = 0; + stream->clock_offset_valid = false; + stream->frame_min = stream->latency; +} + +static void t2audio_reset_clock(struct t2audio_device *a) +{ + unsigned long flags; + + spin_lock_irqsave(&a->clock_lock, flags); + a->clock_offset_ns = 0; + a->clock_samples = 0; + a->clock_offset_valid = false; + spin_unlock_irqrestore(&a->clock_lock, flags); +} + +static void t2audio_reset_streams(struct t2audio_device *a) +{ + struct t2audio_subdevice *sdev; + size_t i; + + list_for_each_entry(sdev, &a->subdevice_list, list) { + for (i = 0; i < sdev->in_stream_cnt; i++) + t2audio_reset_stream(&sdev->in_streams[i]); + for (i = 0; i < sdev->out_stream_cnt; i++) + t2audio_reset_stream(&sdev->out_streams[i]); + } +} + +static int t2audio_init_cmd(struct t2audio_device *a) +{ + int status; + struct t2audio_send_ctx sctx; + struct t2audio_msg buf; + u64 dev_cnt, dev_i; + t2audio_device_id_t *dev_l; + + if ((status = t2audio_send(a, &sctx, 500, + t2audio_msg_write_alive_notification, 1, 3))) { + dev_err(a->dev, "Sending alive notification failed\n"); + return status; + } + + if (wait_for_completion_timeout(&a->remote_alive, msecs_to_jiffies(500)) == 0) { + dev_err(a->dev, "Timed out waiting for remote\n"); + return -ETIMEDOUT; + } + pr_debug("t2bce_audio: Continuing init\n"); + + buf = t2audio_reply_alloc(); + if ((status = t2audio_cmd_get_device_list(a, &buf, &dev_l, &dev_cnt))) { + dev_err(a->dev, "Failed to get device list\n"); + t2audio_reply_free(&buf); + return status; + } + for (dev_i = 0; dev_i < dev_cnt; ++dev_i) + t2audio_init_dev(a, dev_l[dev_i]); + t2audio_reply_free(&buf); + + return 0; +} + +static void t2audio_init_stream_info(struct t2audio_subdevice *sdev, struct t2audio_stream *strm); +static void t2audio_handle_jack_connection_change(struct t2audio_subdevice *sdev); + +static void t2audio_init_dev(struct t2audio_device *a, t2audio_device_id_t dev_id) +{ + struct t2audio_subdevice *sdev; + struct t2audio_msg buf = t2audio_reply_alloc(); + u64 uid_len, stream_cnt, i; + t2audio_object_id_t *stream_list; + char *uid; + + sdev = kzalloc(sizeof(struct t2audio_subdevice), GFP_KERNEL); + + if (t2audio_cmd_get_property(a, &buf, dev_id, dev_id, T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_GLOBAL, T2AUDIO_PROP_UID, 0), + NULL, 0, (void **) &uid, &uid_len) || uid_len > T2AUDIO_DEVICE_MAX_UID_LEN) { + dev_err(a->dev, "Failed to get device uid for device %llx\n", dev_id); + goto fail; + } + pr_debug("t2bce_audio: Remote device %llx %.*s\n", dev_id, (int) uid_len, uid); + + sdev->a = a; + INIT_LIST_HEAD(&sdev->list); + sdev->dev_id = dev_id; + sdev->buf_id = T2AUDIO_BUFFER_ID_NONE; + strscpy(sdev->uid, uid); + + if (t2audio_cmd_get_primitive_property(a, dev_id, dev_id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_INPUT, T2AUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->in_latency, sizeof(u32))) + dev_warn(a->dev, "Failed to query device input latency\n"); + if (t2audio_cmd_get_primitive_property(a, dev_id, dev_id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_OUTPUT, T2AUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->out_latency, sizeof(u32))) + dev_warn(a->dev, "Failed to query device output latency\n"); + + if (t2audio_cmd_get_input_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { + dev_err(a->dev, "Failed to get input stream list for device %llx\n", dev_id); + goto fail; + } + if (stream_cnt > T2AUDIO_DEVICE_MAX_INPUT_STREAMS) { + dev_warn(a->dev, "Device %s input stream count %llu is larger than the supported count of %u\n", + sdev->uid, stream_cnt, T2AUDIO_DEVICE_MAX_INPUT_STREAMS); + stream_cnt = T2AUDIO_DEVICE_MAX_INPUT_STREAMS; + } + sdev->in_stream_cnt = stream_cnt; + for (i = 0; i < stream_cnt; i++) { + sdev->in_streams[i].id = stream_list[i]; + sdev->in_streams[i].buffer_cnt = 0; + t2audio_init_stream_info(sdev, &sdev->in_streams[i]); + sdev->in_streams[i].latency += sdev->in_latency; + } + + if (t2audio_cmd_get_output_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) { + dev_err(a->dev, "Failed to get output stream list for device %llx\n", dev_id); + goto fail; + } + if (stream_cnt > T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS) { + dev_warn(a->dev, "Device %s output stream count %llu is larger than the supported count of %u\n", + sdev->uid, stream_cnt, T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS); + stream_cnt = T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS; + } + sdev->out_stream_cnt = stream_cnt; + for (i = 0; i < stream_cnt; i++) { + sdev->out_streams[i].id = stream_list[i]; + sdev->out_streams[i].buffer_cnt = 0; + t2audio_init_stream_info(sdev, &sdev->out_streams[i]); + sdev->out_streams[i].latency += sdev->out_latency; + } + + if (sdev->is_pcm) + t2audio_create_pcm(sdev); + /* Headphone Jack status */ + if (!strcmp(sdev->uid, "Codec Output")) { + if (snd_jack_new(a->card, sdev->uid, SND_JACK_HEADPHONE, &sdev->jack, true, false)) + dev_warn(a->dev, "Failed to create an attached jack for %s\n", sdev->uid); + t2audio_cmd_property_listener(a, sdev->dev_id, sdev->dev_id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_OUTPUT, T2AUDIO_PROP_JACK_PLUGGED, 0)); + t2audio_handle_jack_connection_change(sdev); + } + + t2audio_reply_free(&buf); + + list_add_tail(&sdev->list, &a->subdevice_list); + return; + +fail: + t2audio_reply_free(&buf); + kfree(sdev); +} + +static void t2audio_init_stream_info(struct t2audio_subdevice *sdev, struct t2audio_stream *strm) +{ + if (t2audio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_GLOBAL, T2AUDIO_PROP_PHYS_FORMAT, 0), NULL, 0, + &strm->desc, sizeof(strm->desc))) + dev_warn(sdev->a->dev, "Failed to query stream descriptor\n"); + if (t2audio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_GLOBAL, T2AUDIO_PROP_LATENCY, 0), NULL, 0, &strm->latency, sizeof(u32))) + dev_warn(sdev->a->dev, "Failed to query stream latency\n"); + if (strm->desc.format_id == T2AUDIO_FORMAT_LPCM) + sdev->is_pcm = true; +} + +static void t2audio_free_dev(struct t2audio_subdevice *sdev) +{ + size_t i; + for (i = 0; i < sdev->in_stream_cnt; i++) { + if (sdev->in_streams[i].alsa_hw_desc) + kfree(sdev->in_streams[i].alsa_hw_desc); + if (sdev->in_streams[i].buffers) + kfree(sdev->in_streams[i].buffers); + } + for (i = 0; i < sdev->out_stream_cnt; i++) { + if (sdev->out_streams[i].alsa_hw_desc) + kfree(sdev->out_streams[i].alsa_hw_desc); + if (sdev->out_streams[i].buffers) + kfree(sdev->out_streams[i].buffers); + } + kfree(sdev); +} + +static struct t2audio_subdevice *t2audio_find_dev_by_dev_id(struct t2audio_device *a, t2audio_device_id_t dev_id) +{ + struct t2audio_subdevice *sdev; + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (dev_id == sdev->dev_id) + return sdev; + } + return NULL; +} + +static struct t2audio_subdevice *t2audio_find_dev_by_uid(struct t2audio_device *a, const char *uid) +{ + struct t2audio_subdevice *sdev; + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (!strcmp(uid, sdev->uid)) + return sdev; + } + return NULL; +} + +static void t2audio_init_bs_stream(struct t2audio_device *a, struct t2audio_stream *strm, + struct t2audio_buffer_struct_stream *bs_strm); +static void t2audio_init_bs_stream_host(struct t2audio_device *a, struct t2audio_stream *strm, + struct t2audio_buffer_struct_stream *bs_strm); + +static int t2audio_init_bs(struct t2audio_device *a) +{ + int i, j; + struct t2audio_buffer_struct_device *dev; + struct t2audio_subdevice *sdev; + u32 ver, sig, bs_base; + + ver = ioread32(&a->reg_mem_gpr[0]); + if (ver < 3) { + dev_err(a->dev, "t2bce_audio: Bad GPR version (%u)", ver); + return -EINVAL; + } + sig = ioread32(&a->reg_mem_gpr[1]); + if (sig != T2AUDIO_SIG) { + dev_err(a->dev, "t2bce_audio: Bad GPR sig (%x)", sig); + return -EINVAL; + } + bs_base = ioread32(&a->reg_mem_gpr[2]); + a->bs = (struct t2audio_buffer_struct *) ((u8 *) a->reg_mem_bs + bs_base); + if (a->bs->signature != T2AUDIO_SIG) { + dev_err(a->dev, "t2bce_audio: Bad BufferStruct sig (%x)", a->bs->signature); + return -EINVAL; + } + pr_debug("t2bce_audio: BufferStruct ver = %i\n", a->bs->version); + pr_debug("t2bce_audio: Num devices = %i\n", a->bs->num_devices); + for (i = 0; i < a->bs->num_devices; i++) { + dev = &a->bs->devices[i]; + pr_debug("t2bce_audio: Device %i %s\n", i, dev->name); + + sdev = t2audio_find_dev_by_uid(a, dev->name); + if (!sdev) { + dev_err(a->dev, "t2bce_audio: Subdevice not found for BufferStruct device %s\n", dev->name); + continue; + } + sdev->buf_id = (u8) i; + dev->num_input_streams = 0; + for (j = 0; j < dev->num_output_streams; j++) { + pr_debug("t2bce_audio: Device %i Stream %i: Output; Buffer Count = %i\n", i, j, + dev->output_streams[j].num_buffers); + if (j < sdev->out_stream_cnt) + t2audio_init_bs_stream(a, &sdev->out_streams[j], &dev->output_streams[j]); + } + } + + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (sdev->buf_id != T2AUDIO_BUFFER_ID_NONE) + continue; + sdev->buf_id = i; + pr_debug("t2bce_audio: Created device %i %s\n", i, sdev->uid); + strcpy(a->bs->devices[i].name, sdev->uid); + a->bs->devices[i].num_input_streams = 0; + a->bs->devices[i].num_output_streams = 0; + a->bs->num_devices = ++i; + } + list_for_each_entry(sdev, &a->subdevice_list, list) { + if (sdev->in_stream_cnt == 1) { + pr_debug("t2bce_audio: Device %i Host Stream; Input\n", sdev->buf_id); + t2audio_init_bs_stream_host(a, &sdev->in_streams[0], &a->bs->devices[sdev->buf_id].input_streams[0]); + a->bs->devices[sdev->buf_id].num_input_streams = 1; + wmb(); + + if (t2audio_cmd_set_input_stream_address_ranges(a, sdev->dev_id)) { + dev_err(a->dev, "t2bce_audio: Failed to set input stream address ranges\n"); + } + } + } + + return 0; +} + +static void t2audio_init_bs_stream(struct t2audio_device *a, struct t2audio_stream *strm, + struct t2audio_buffer_struct_stream *bs_strm) +{ + size_t i; + strm->buffer_cnt = bs_strm->num_buffers; + if (bs_strm->num_buffers > T2AUDIO_DEVICE_MAX_BUFFER_COUNT) { + dev_warn(a->dev, "BufferStruct buffer count %u exceeds driver limit of %u\n", bs_strm->num_buffers, + T2AUDIO_DEVICE_MAX_BUFFER_COUNT); + strm->buffer_cnt = T2AUDIO_DEVICE_MAX_BUFFER_COUNT; + } + if (!strm->buffer_cnt) + return; + strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct t2audio_dma_buf), GFP_KERNEL); + if (!strm->buffers) { + dev_err(a->dev, "Buffer list allocation failed\n"); + return; + } + for (i = 0; i < strm->buffer_cnt; i++) { + strm->buffers[i].dma_addr = a->reg_mem_bs_dma + (dma_addr_t) bs_strm->buffers[i].address; + strm->buffers[i].ptr = a->reg_mem_bs + bs_strm->buffers[i].address; + strm->buffers[i].size = bs_strm->buffers[i].size; + strm->buffers[i].type = T2AUDIO_DMA_BUF_IOMEM; + } + + if (strm->buffer_cnt == 1) { + strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); + if (t2audio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { + kfree(strm->alsa_hw_desc); + strm->alsa_hw_desc = NULL; + } + } +} + +static void t2audio_init_bs_stream_host(struct t2audio_device *a, struct t2audio_stream *strm, + struct t2audio_buffer_struct_stream *bs_strm) +{ + size_t size; + dma_addr_t dma_addr; + void *dma_ptr; + size = strm->desc.bytes_per_packet * 16640; + dma_ptr = dma_alloc_coherent(&a->pci->dev, size, &dma_addr, GFP_KERNEL); + if (!dma_ptr) { + dev_err(a->dev, "dma_alloc_coherent failed\n"); + return; + } + bs_strm->buffers[0].address = dma_addr; + bs_strm->buffers[0].size = size; + bs_strm->num_buffers = 1; + + memset(dma_ptr, 0, size); + + strm->buffer_cnt = 1; + strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct t2audio_dma_buf), GFP_KERNEL); + if (!strm->buffers) { + dev_err(a->dev, "Buffer list allocation failed\n"); + return; + } + strm->buffers[0].dma_addr = dma_addr; + strm->buffers[0].ptr = dma_ptr; + strm->buffers[0].size = size; + strm->buffers[0].type = T2AUDIO_DMA_BUF_COHERENT; + + strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL); + if (t2audio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) { + kfree(strm->alsa_hw_desc); + strm->alsa_hw_desc = NULL; + } +} + +static void t2audio_handle_prop_change(struct t2audio_device *a, struct t2audio_msg *msg); + +void t2audio_handle_notification(struct t2audio_device *a, struct t2audio_msg *msg) +{ + struct t2audio_send_ctx sctx; + struct t2audio_msg_base base; + if (t2audio_msg_read_base(msg, &base)) + return; + switch (base.msg) { + case T2AUDIO_MSG_NOTIFICATION_BOOT: + pr_debug("t2bce_audio: Received boot notification from remote\n"); + + /* Resend the alive notify */ + if (t2audio_send(a, &sctx, 500, + t2audio_msg_write_alive_notification, 1, 3)) { + pr_err("Sending alive notification failed\n"); + } + break; + case T2AUDIO_MSG_NOTIFICATION_ALIVE: + pr_debug("t2bce_audio: Received alive notification from remote\n"); + complete_all(&a->remote_alive); + break; + case T2AUDIO_MSG_PROPERTY_CHANGED: + t2audio_handle_prop_change(a, msg); + break; + default: + pr_debug("t2bce_audio: Unhandled notification %i\n", base.msg); + break; + } +} + +struct t2audio_prop_change_work_struct { + struct work_struct ws; + struct t2audio_device *a; + t2audio_device_id_t dev; + t2audio_object_id_t obj; + struct t2audio_prop_addr prop; +}; + +static void t2audio_handle_jack_connection_change(struct t2audio_subdevice *sdev) +{ + u32 plugged; + if (!sdev->jack) + return; + /* NOTE: Apple made the plug status scoped to the input and output streams. This makes no sense for us, so I just + * always pick the OUTPUT status. */ + if (t2audio_cmd_get_primitive_property(sdev->a, sdev->dev_id, sdev->dev_id, + T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_OUTPUT, T2AUDIO_PROP_JACK_PLUGGED, 0), NULL, 0, &plugged, sizeof(plugged))) { + dev_err(sdev->a->dev, "Failed to get jack enable status\n"); + return; + } + pr_debug("t2bce_audio: Jack is now %s\n", plugged ? "plugged" : "unplugged"); + snd_jack_report(sdev->jack, plugged ? sdev->jack->type : 0); +} + +void t2audio_handle_prop_change_work(struct work_struct *ws) +{ + struct t2audio_prop_change_work_struct *work = container_of(ws, struct t2audio_prop_change_work_struct, ws); + struct t2audio_subdevice *sdev; + + sdev = t2audio_find_dev_by_dev_id(work->a, work->dev); + if (!sdev) { + dev_err(work->a->dev, "Property notification change: device not found\n"); + goto done; + } + pr_debug("t2bce_audio: Property changed for device: %s\n", sdev->uid); + + if (work->prop.scope == T2AUDIO_PROP_SCOPE_OUTPUT && work->prop.selector == T2AUDIO_PROP_JACK_PLUGGED) { + t2audio_handle_jack_connection_change(sdev); + } + +done: + kfree(work); +} + +void t2audio_handle_prop_change(struct t2audio_device *a, struct t2audio_msg *msg) +{ + /* NOTE: This is a scheduled work because this callback will generally need to query device information and this + * is not possible when we are in the reply parsing code's context. */ + struct t2audio_prop_change_work_struct *work; + work = kmalloc(sizeof(struct t2audio_prop_change_work_struct), GFP_KERNEL); + work->a = a; + INIT_WORK(&work->ws, t2audio_handle_prop_change_work); + t2audio_msg_read_property_changed(msg, &work->dev, &work->obj, &work->prop); + schedule_work(&work->ws); +} + +#define t2audio_send_cmd_response(a, sctx, msg, fn, ...) \ + if (t2audio_send_with_tag(a, sctx, ((struct t2audio_msg_header *) msg->data)->tag, 500, fn, ##__VA_ARGS__)) \ + pr_err("t2bce_audio: Failed to reply to a command\n"); + +void t2audio_handle_cmd_timestamp(struct t2audio_device *a, struct t2audio_msg *msg) +{ + ktime_t time_os = ktime_get_boottime(); + unsigned long flags; + struct t2audio_send_ctx sctx; + struct t2audio_subdevice *sdev; + bool clock_ready; + s64 clock_offset_ns; + s64 offset_sample; + u64 devid, timestamp, update_seed; + t2audio_msg_read_update_timestamp(msg, &devid, ×tamp, &update_seed); + + offset_sample = ktime_to_ns(time_os) - (s64)timestamp; + spin_lock_irqsave(&a->clock_lock, flags); + if (offset_sample >= 0) { + if (!a->clock_offset_valid || offset_sample < a->clock_offset_ns) { + a->clock_offset_ns = offset_sample; + a->clock_offset_valid = true; + } + if (a->clock_samples < 2) + a->clock_samples++; + } + clock_offset_ns = a->clock_offset_ns; + clock_ready = a->clock_samples >= 2; + spin_unlock_irqrestore(&a->clock_lock, flags); + pr_debug("t2bce_audio: timestamp dev=%llx t2=%llx host=%lld seed=%llx sample_offset=%lld clock_offset=%lld\n", + devid, timestamp, ktime_to_ns(time_os), update_seed, + offset_sample, clock_offset_ns); + + sdev = t2audio_find_dev_by_dev_id(a, devid); + if (sdev) + t2audio_handle_timestamp(sdev, timestamp, update_seed, clock_offset_ns, + clock_ready); + + t2audio_send_cmd_response(a, &sctx, msg, + t2audio_msg_write_update_timestamp_response); +} + +void t2audio_handle_command(struct t2audio_device *a, struct t2audio_msg *msg) +{ + struct t2audio_msg_base base; + if (t2audio_msg_read_base(msg, &base)) + return; + switch (base.msg) { + case T2AUDIO_MSG_UPDATE_TIMESTAMP: + t2audio_handle_cmd_timestamp(a, msg); + break; + default: + pr_debug("t2bce_audio: Unhandled device command %i\n", base.msg); + break; + } +} + +static struct pci_device_id t2audio_ids[ ] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1803) }, + { 0, }, +}; +MODULE_DEVICE_TABLE(pci, t2audio_ids); + +struct dev_pm_ops t2audio_pci_driver_pm = { + .suspend = t2audio_suspend, + .resume = t2audio_resume +}; +struct pci_driver t2audio_pci_driver = { + .name = "t2bce_audio", + .id_table = t2audio_ids, + .probe = t2audio_probe, + .remove = t2audio_remove, + .shutdown = t2audio_shutdown, + .driver = { + .pm = &t2audio_pci_driver_pm + } +}; + + +static int __init t2audio_module_init(void) +{ + int result; + if ((result = alloc_chrdev_region(&t2audio_chrdev, 0, 1, "t2bce_audio"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + t2audio_class = class_create(THIS_MODULE, "t2bce_audio"); +#else + t2audio_class = class_create("t2bce_audio"); +#endif + if (IS_ERR(t2audio_class)) { + result = PTR_ERR(t2audio_class); + goto fail_class; + } + + result = pci_register_driver(&t2audio_pci_driver); + if (result) + goto fail_drv; + pr_info("t2bce_audio: module initialized\n"); + return 0; + +fail_drv: + pci_unregister_driver(&t2audio_pci_driver); +fail_class: + class_destroy(t2audio_class); +fail_chrdev: + unregister_chrdev_region(t2audio_chrdev, 1); + if (!result) + result = -EINVAL; + pr_info("t2bce_audio: module init failed status=%d\n", result); + return result; +} + +static void __exit t2audio_module_exit(void) +{ + pci_unregister_driver(&t2audio_pci_driver); + class_destroy(t2audio_class); + unregister_chrdev_region(t2audio_chrdev, 1); + pr_info("t2bce_audio: module exited\n"); +} + +struct t2audio_alsa_pcm_id_mapping t2audio_alsa_id_mappings[] = { + {"Speaker", 0}, + {"Digital Mic", 1}, + {"Codec Output", 2}, + {"Codec Input", 3}, + {"Bridge Loopback", 4}, + {} +}; + +module_param_named(index, t2audio_alsa_index, int, 0444); +MODULE_PARM_DESC(index, "Index value for Apple Internal Audio soundcard."); +module_param_named(id, t2audio_alsa_id, charp, 0444); +MODULE_PARM_DESC(id, "ID string for Apple Internal Audio soundcard."); +MODULE_SOFTDEP("pre: t2bce_core"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("AndrĂ© Eikmeyer "); +MODULE_DESCRIPTION("Apple T2 Audio Driver"); +MODULE_VERSION("0.01"); +module_init(t2audio_module_init); +module_exit(t2audio_module_exit); diff --git a/drivers/staging/t2bce/t2bce_audio/audio.h b/drivers/staging/t2bce/t2bce_audio/audio.h new file mode 100644 index 000000000..6eb57789e --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/audio.h @@ -0,0 +1,148 @@ +#ifndef T2AUDIO_H +#define T2AUDIO_H + +#include +#include +#include +#include "t2bce_core_transport.h" +#include "protocol_bce.h" +#include "description.h" + +#define T2AUDIO_SIG 0x19870423 + +#define T2AUDIO_DEVICE_MAX_UID_LEN 128 +#define T2AUDIO_DEVICE_MAX_INPUT_STREAMS 1 +#define T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS 1 +#define T2AUDIO_DEVICE_MAX_BUFFER_COUNT 1 + +#define T2AUDIO_BUFFER_ID_NONE 0xffu + +struct snd_card; +struct snd_pcm; +struct snd_pcm_hardware; +struct snd_jack; + +struct __attribute__((packed)) __attribute__((aligned(4))) t2audio_buffer_struct_buffer { + size_t address; + size_t size; + size_t pad[4]; +}; +struct t2audio_buffer_struct_stream { + u8 num_buffers; + struct t2audio_buffer_struct_buffer buffers[100]; + char filler[32]; +}; +struct t2audio_buffer_struct_device { + char name[128]; + u8 num_input_streams; + u8 num_output_streams; + struct t2audio_buffer_struct_stream input_streams[5]; + struct t2audio_buffer_struct_stream output_streams[5]; + char filler[128]; +}; +struct t2audio_buffer_struct { + u32 version; + u32 signature; + u32 flags; + u8 num_devices; + struct t2audio_buffer_struct_device devices[20]; +}; + +struct t2audio_device; + +struct t2audio_deferred_msg { + struct work_struct ws; + struct t2audio_device *a; + struct t2audio_msg msg; +}; + +enum t2audio_dma_buf_type { + T2AUDIO_DMA_BUF_IOMEM, + T2AUDIO_DMA_BUF_COHERENT, +}; + +struct t2audio_dma_buf { + dma_addr_t dma_addr; + void *ptr; + size_t size; + enum t2audio_dma_buf_type type; +}; +struct t2audio_stream { + t2audio_object_id_t id; + size_t buffer_cnt; + struct t2audio_dma_buf *buffers; + + struct t2audio_apple_description desc; + struct snd_pcm_hardware *alsa_hw_desc; + u32 latency; + + bool waiting_for_first_ts; + + ktime_t remote_timestamp; + ktime_t timestamp_accept_after; + s64 clock_offset_ns; + u64 timestamp_seed; + bool clock_offset_valid; + snd_pcm_sframes_t frame_min; + int started; +}; +struct t2audio_subdevice { + struct t2audio_device *a; + struct list_head list; + t2audio_device_id_t dev_id; + u32 in_latency, out_latency; + u8 buf_id; + int alsa_id; + char uid[T2AUDIO_DEVICE_MAX_UID_LEN + 1]; + size_t in_stream_cnt; + struct t2audio_stream in_streams[T2AUDIO_DEVICE_MAX_INPUT_STREAMS]; + size_t out_stream_cnt; + struct t2audio_stream out_streams[T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS]; + bool is_pcm; + struct snd_pcm *pcm; + struct snd_jack *jack; +}; +struct t2audio_alsa_pcm_id_mapping { + const char *name; + int alsa_id; +}; + +struct t2audio_device { + struct pci_dev *pci; + dev_t devt; + struct device *dev; + void __iomem *reg_mem_bs; + dma_addr_t reg_mem_bs_dma; + void __iomem *reg_mem_cfg; + + u32 __iomem *reg_mem_gpr; + + struct t2audio_buffer_struct *bs; + + struct t2bce_core_client *bce; + struct t2audio_bce bcem; + + struct snd_card *card; + + struct list_head subdevice_list; + int next_alsa_id; + + struct completion remote_alive; + struct work_struct resume_work; + bool resume_deferred; + bool pm_quiesced; + + spinlock_t clock_lock; + s64 clock_offset_ns; + u32 clock_samples; + bool clock_offset_valid; +}; + +void t2audio_handle_notification(struct t2audio_device *a, struct t2audio_msg *msg); +void t2audio_handle_prop_change_work(struct work_struct *ws); +void t2audio_handle_cmd_timestamp(struct t2audio_device *a, struct t2audio_msg *msg); +void t2audio_handle_command(struct t2audio_device *a, struct t2audio_msg *msg); + +extern struct t2audio_alsa_pcm_id_mapping t2audio_alsa_id_mappings[]; + +#endif //T2AUDIO_H diff --git a/drivers/staging/t2bce/t2bce_audio/description.h b/drivers/staging/t2bce/t2bce_audio/description.h new file mode 100644 index 000000000..011ebfd69 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/description.h @@ -0,0 +1,42 @@ +#ifndef T2AUDIO_DESCRIPTION_H +#define T2AUDIO_DESCRIPTION_H + +#include + +struct t2audio_apple_description { + u64 sample_rate_double; + u32 format_id; + u32 format_flags; + u32 bytes_per_packet; + u32 frames_per_packet; + u32 bytes_per_frame; + u32 channels_per_frame; + u32 bits_per_channel; + u32 reserved; +}; + +enum { + T2AUDIO_FORMAT_LPCM = 0x6c70636d // 'lpcm' +}; + +enum { + T2AUDIO_FORMAT_FLAG_FLOAT = 1, + T2AUDIO_FORMAT_FLAG_BIG_ENDIAN = 2, + T2AUDIO_FORMAT_FLAG_SIGNED = 4, + T2AUDIO_FORMAT_FLAG_PACKED = 8, + T2AUDIO_FORMAT_FLAG_ALIGNED_HIGH = 16, + T2AUDIO_FORMAT_FLAG_NON_INTERLEAVED = 32, + T2AUDIO_FORMAT_FLAG_NON_MIXABLE = 64 +}; + +static inline u64 t2audio_double_to_u64(u64 d) +{ + u8 sign = (u8) ((d >> 63) & 1); + s32 exp = (s32) ((d >> 52) & 0x7ff) - 1023; + u64 fr = d & ((1LL << 52) - 1); + if (sign || exp < 0) + return 0; + return (u64) ((1LL << exp) + (fr >> (52 - exp))); +} + +#endif //T2AUDIO_DESCRIPTION_H diff --git a/drivers/staging/t2bce/t2bce_audio/pcm.c b/drivers/staging/t2bce/t2bce_audio/pcm.c new file mode 100644 index 000000000..dcffd53a7 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/pcm.c @@ -0,0 +1,520 @@ +#include "pcm.h" +#include "audio.h" +#include +#include +#include + +static u64 t2audio_get_alsa_fmtbit(struct t2audio_apple_description *desc) +{ + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_FLOAT) { + if (desc->bits_per_channel == 32) { + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_BIG_ENDIAN) + return SNDRV_PCM_FMTBIT_FLOAT_BE; + else + return SNDRV_PCM_FMTBIT_FLOAT_LE; + } else if (desc->bits_per_channel == 64) { + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_BIG_ENDIAN) + return SNDRV_PCM_FMTBIT_FLOAT64_BE; + else + return SNDRV_PCM_FMTBIT_FLOAT64_LE; + } else { + pr_err("t2bce_audio: unsupported bits per channel for float format: %u\n", desc->bits_per_channel); + return 0; + } + } +#define DEFINE_BPC_OPTION(val, b) \ + case val: \ + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_BIG_ENDIAN) { \ + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_SIGNED) \ + return SNDRV_PCM_FMTBIT_S ## b ## BE; \ + else \ + return SNDRV_PCM_FMTBIT_U ## b ## BE; \ + } else { \ + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_SIGNED) \ + return SNDRV_PCM_FMTBIT_S ## b ## LE; \ + else \ + return SNDRV_PCM_FMTBIT_U ## b ## LE; \ + } + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_PACKED) { + switch (desc->bits_per_channel) { + case 8: + case 16: + case 32: + break; + DEFINE_BPC_OPTION(24, 24_3) + default: + pr_err("t2bce_audio: unsupported bits per channel for packed format: %u\n", desc->bits_per_channel); + return 0; + } + } + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_ALIGNED_HIGH) { + switch (desc->bits_per_channel) { + DEFINE_BPC_OPTION(24, 32_) + default: + pr_err("t2bce_audio: unsupported bits per channel for high-aligned format: %u\n", desc->bits_per_channel); + return 0; + } + } + switch (desc->bits_per_channel) { + case 8: + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_SIGNED) + return SNDRV_PCM_FMTBIT_S8; + else + return SNDRV_PCM_FMTBIT_U8; + DEFINE_BPC_OPTION(16, 16_) + DEFINE_BPC_OPTION(24, 24_) + DEFINE_BPC_OPTION(32, 32_) + default: + pr_err("t2bce_audio: unsupported bits per channel: %u\n", desc->bits_per_channel); + return 0; + } +} +int t2audio_create_hw_info(struct t2audio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, + size_t buf_size) +{ + uint rate; + alsa_hw->info = (SNDRV_PCM_INFO_MMAP | + SNDRV_PCM_INFO_BLOCK_TRANSFER | + SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_NO_PERIOD_WAKEUP | + SNDRV_PCM_INFO_DOUBLE); + if (desc->format_flags & T2AUDIO_FORMAT_FLAG_NON_MIXABLE) + pr_warn("t2bce_audio: unsupported hw flag: NON_MIXABLE\n"); + if (!(desc->format_flags & T2AUDIO_FORMAT_FLAG_NON_INTERLEAVED)) + alsa_hw->info |= SNDRV_PCM_INFO_INTERLEAVED; + alsa_hw->formats = t2audio_get_alsa_fmtbit(desc); + if (!alsa_hw->formats) + return -EINVAL; + rate = (uint) t2audio_double_to_u64(desc->sample_rate_double); + alsa_hw->rates = snd_pcm_rate_to_rate_bit(rate); + alsa_hw->rate_min = rate; + alsa_hw->rate_max = rate; + alsa_hw->channels_min = desc->channels_per_frame; + alsa_hw->channels_max = desc->channels_per_frame; + alsa_hw->buffer_bytes_max = buf_size; + alsa_hw->period_bytes_min = desc->bytes_per_packet; + alsa_hw->period_bytes_max = desc->bytes_per_packet; + alsa_hw->periods_min = (uint) (buf_size / desc->bytes_per_packet); + alsa_hw->periods_max = (uint) (buf_size / desc->bytes_per_packet); + pr_debug("t2audio_create_hw_info: format = %llu, rate = %u/%u. channels = %u, periods = %u, period size = %lu\n", + alsa_hw->formats, alsa_hw->rate_min, alsa_hw->rates, alsa_hw->channels_min, alsa_hw->periods_min, + alsa_hw->period_bytes_min); + return 0; +} + +static struct t2audio_stream *t2audio_pcm_stream(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + return &sdev->out_streams[substream->number]; + else + return &sdev->in_streams[substream->number]; +} + +static struct t2audio_dma_buf *t2audio_pcm_dma_buf(struct t2audio_stream *stream) +{ + if (!stream->buffer_cnt || !stream->buffers) + return NULL; + + return &stream->buffers[0]; +} + +static void t2audio_dma_memset(struct t2audio_dma_buf *buf, size_t offset, int value, size_t size) +{ + if (!buf || offset >= buf->size) + return; + + size = min(size, buf->size - offset); + switch (buf->type) { + case T2AUDIO_DMA_BUF_IOMEM: + memset_io((u8 __iomem *) buf->ptr + offset, value, size); + break; + case T2AUDIO_DMA_BUF_COHERENT: + memset((u8 *) buf->ptr + offset, value, size); + break; + } +} + +static void t2audio_dma_copy_from(struct t2audio_dma_buf *buf, void *dst, size_t offset, size_t size) +{ + if (!buf || offset >= buf->size) + return; + + size = min(size, buf->size - offset); + switch (buf->type) { + case T2AUDIO_DMA_BUF_IOMEM: + memcpy_fromio(dst, (u8 __iomem *) buf->ptr + offset, size); + break; + case T2AUDIO_DMA_BUF_COHERENT: + memcpy(dst, (u8 *) buf->ptr + offset, size); + break; + } +} + +static void t2audio_dma_copy_to(struct t2audio_dma_buf *buf, size_t offset, const void *src, size_t size) +{ + if (!buf || offset >= buf->size) + return; + + size = min(size, buf->size - offset); + switch (buf->type) { + case T2AUDIO_DMA_BUF_IOMEM: + memcpy_toio((u8 __iomem *) buf->ptr + offset, src, size); + break; + case T2AUDIO_DMA_BUF_COHERENT: + memcpy((u8 *) buf->ptr + offset, src, size); + break; + } +} + +static void t2audio_pcm_zero_frames(struct snd_pcm_substream *substream, + snd_pcm_uframes_t first, snd_pcm_uframes_t frames) +{ + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + struct t2audio_dma_buf *buf = t2audio_pcm_dma_buf(stream); + struct snd_pcm_runtime *runtime = substream->runtime; + size_t offset; + size_t size; + + if (!buf || !runtime || !runtime->buffer_size || !frames) + return; + + first %= runtime->buffer_size; + if (frames > runtime->buffer_size) + frames = runtime->buffer_size; + + offset = frames_to_bytes(runtime, first); + if (first + frames <= runtime->buffer_size) { + size = frames_to_bytes(runtime, frames); + t2audio_dma_memset(buf, offset, 0, size); + return; + } + + size = frames_to_bytes(runtime, runtime->buffer_size - first); + t2audio_dma_memset(buf, offset, 0, size); + t2audio_dma_memset(buf, 0, 0, frames_to_bytes(runtime, frames - (runtime->buffer_size - first))); +} + +static int t2audio_pcm_open(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + + pr_debug("t2bce_audio: pcm open dev=%s direction=%s substream=%u\n", + sdev->uid, + substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture", + substream->number); + substream->runtime->hw = *t2audio_pcm_stream(substream)->alsa_hw_desc; + + return 0; +} + +static int t2audio_pcm_close(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + + pr_debug("t2bce_audio: pcm close dev=%s direction=%s substream=%u\n", + sdev->uid, + substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture", + substream->number); + return 0; +} + +static int t2audio_pcm_prepare(struct snd_pcm_substream *substream) +{ + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + + stream->waiting_for_first_ts = true; + stream->remote_timestamp = 0; + stream->timestamp_accept_after = 0; + stream->clock_offset_ns = 0; + stream->timestamp_seed = 0; + stream->clock_offset_valid = false; + stream->frame_min = stream->latency; + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && substream->runtime->buffer_size) + t2audio_pcm_zero_frames(substream, 0, substream->runtime->buffer_size); + + return 0; +} + +static int t2audio_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *hw_params) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct t2audio_stream *astream = t2audio_pcm_stream(substream); + + pr_debug("t2bce_audio: pcm hw_params dev=%s direction=%s substream=%u\n", + sdev->uid, + substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture", + substream->number); + + if (!astream->buffer_cnt || !astream->buffers) + return -EINVAL; + + substream->runtime->dma_area = astream->buffers[0].ptr; + substream->runtime->dma_addr = astream->buffers[0].dma_addr; + substream->runtime->dma_bytes = astream->buffers[0].size; + return 0; +} + +static int t2audio_pcm_hw_free(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + + pr_debug("t2bce_audio: pcm hw_free dev=%s direction=%s substream=%u\n", + sdev->uid, + substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture", + substream->number); + return 0; +} + +static int t2audio_pcm_start(struct snd_pcm_substream *substream) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + struct t2audio_dma_buf *dmabuf = t2audio_pcm_dma_buf(stream); + void *buf = NULL; + size_t s = 0; + ktime_t time_start, time_end; + bool back_buffer; + int status; + + time_start = ktime_get(); + smp_store_release(&stream->started, 0); + + back_buffer = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK); + + if (back_buffer) { + snd_pcm_uframes_t appl_ptr; + + if (!dmabuf) + return -EINVAL; + if (!substream->runtime->buffer_size) + return -EINVAL; + + appl_ptr = substream->runtime->control->appl_ptr % substream->runtime->buffer_size; + s = frames_to_bytes(substream->runtime, appl_ptr); + if (s) { + buf = kmalloc(s, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + t2audio_dma_copy_from(dmabuf, buf, 0, s); + time_end = ktime_get(); + pr_debug("t2bce_audio: Backed up the buffer in %lluns [%li]\n", ktime_to_ns(time_end - time_start), + appl_ptr); + } + } + + status = t2audio_cmd_start_io(sdev->a, sdev->dev_id); + if (back_buffer && buf) + t2audio_dma_copy_to(dmabuf, 0, buf, s); + kfree(buf); + + if (status) + return status; + + /* Timestamps received while START_IO was pending belong to the old epoch. */ + stream->remote_timestamp = 0; + stream->waiting_for_first_ts = true; + stream->timestamp_accept_after = ktime_get_boottime(); + stream->clock_offset_ns = 0; + stream->timestamp_seed = 0; + stream->clock_offset_valid = false; + stream->frame_min = stream->latency; + smp_store_release(&stream->started, 1); + + time_end = ktime_get(); + pr_debug("t2bce_audio: start_io %s %lld us\n", + sdev->uid, ktime_to_us(ktime_sub(time_end, time_start))); + return 0; +} + +static int t2audio_pcm_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + int err; + + /* bridgeOS exposes one ALSA substream per remote stream. */ + if (substream->number != 0) + return 0; + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + pr_debug("t2bce_audio: TRIGGER START %s\n", sdev->uid); + err = t2audio_pcm_start(substream); + if (err) + return err; + break; + case SNDRV_PCM_TRIGGER_STOP: + pr_debug("t2bce_audio: TRIGGER STOP %s\n", sdev->uid); + smp_store_release(&stream->started, 0); + err = t2audio_cmd_stop_io(sdev->a, sdev->dev_id); + stream->remote_timestamp = 0; + stream->waiting_for_first_ts = true; + stream->timestamp_accept_after = 0; + stream->clock_offset_ns = 0; + stream->timestamp_seed = 0; + stream->clock_offset_valid = false; + if (err) + return err; + break; + default: + return -EINVAL; + } + return 0; +} + +static snd_pcm_uframes_t t2audio_pcm_pointer(struct snd_pcm_substream *substream) +{ + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + ktime_t time_from_start; + snd_pcm_sframes_t frames; + snd_pcm_sframes_t buffer_time_length; + + if (!smp_load_acquire(&stream->started) || stream->waiting_for_first_ts) + return 0; + + /* bridgeOS reports coarse timestamps; interpolate the ALSA pointer locally. */ + time_from_start = ktime_get_boottime() - stream->remote_timestamp; + if (ktime_to_ns(time_from_start) < 0) + return 0; + buffer_time_length = NSEC_PER_SEC * substream->runtime->buffer_size / substream->runtime->rate; + frames = (ktime_to_ns(time_from_start) % buffer_time_length) * substream->runtime->buffer_size / buffer_time_length; + if (ktime_to_ns(time_from_start) < buffer_time_length) { + if (frames < stream->frame_min) + frames = stream->frame_min; + else + stream->frame_min = 0; + } else { + if (ktime_to_ns(time_from_start) < 2 * buffer_time_length) + stream->frame_min = frames; + else + stream->frame_min = 0; + } + frames -= stream->latency; + if (frames < 0) + frames += ((-frames - 1) / substream->runtime->buffer_size + 1) * substream->runtime->buffer_size; + frames %= substream->runtime->buffer_size; + return (snd_pcm_uframes_t) frames; +} + +static int t2audio_pcm_mmap(struct snd_pcm_substream *substream, struct vm_area_struct *area) +{ + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + struct t2audio_stream *stream = t2audio_pcm_stream(substream); + struct t2audio_dma_buf *buf; + + if (!stream->buffer_cnt || !stream->buffers) + return -EINVAL; + + buf = &stream->buffers[0]; + switch (buf->type) { + case T2AUDIO_DMA_BUF_IOMEM: + return snd_pcm_lib_mmap_iomem(substream, area); + case T2AUDIO_DMA_BUF_COHERENT: + return dma_mmap_coherent(sdev->a->dev, area, buf->ptr, buf->dma_addr, buf->size); + default: + return -EINVAL; + } +} + +static struct snd_pcm_ops t2audio_pcm_ops = { + .open = t2audio_pcm_open, + .close = t2audio_pcm_close, + .ioctl = snd_pcm_lib_ioctl, + .hw_params = t2audio_pcm_hw_params, + .hw_free = t2audio_pcm_hw_free, + .prepare = t2audio_pcm_prepare, + .trigger = t2audio_pcm_trigger, + .pointer = t2audio_pcm_pointer, + .mmap = t2audio_pcm_mmap +}; + +int t2audio_create_pcm(struct t2audio_subdevice *sdev) +{ + struct snd_pcm *pcm; + struct t2audio_alsa_pcm_id_mapping *id_mapping; + int err; + + if (!sdev->is_pcm || (sdev->in_stream_cnt == 0 && sdev->out_stream_cnt == 0)) { + return -EINVAL; + } + + for (id_mapping = t2audio_alsa_id_mappings; id_mapping->name; id_mapping++) { + if (!strcmp(sdev->uid, id_mapping->name)) { + sdev->alsa_id = id_mapping->alsa_id; + break; + } + } + if (!id_mapping->name) + sdev->alsa_id = sdev->a->next_alsa_id++; + err = snd_pcm_new(sdev->a->card, sdev->uid, sdev->alsa_id, + (int) sdev->out_stream_cnt, (int) sdev->in_stream_cnt, &pcm); + if (err < 0) + return err; + pcm->private_data = sdev; + pcm->nonatomic = 1; + sdev->pcm = pcm; + strcpy(pcm->name, sdev->uid); + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &t2audio_pcm_ops); + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &t2audio_pcm_ops); + return 0; +} + +static void t2audio_handle_stream_timestamp(struct snd_pcm_substream *substream, + u64 dev_timestamp, u64 update_seed, + s64 clock_offset_ns, bool clock_ready) +{ + unsigned long flags; + ktime_t event_timestamp; + struct t2audio_stream *stream; + struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream); + + stream = t2audio_pcm_stream(substream); + snd_pcm_stream_lock_irqsave(substream, flags); + if (!smp_load_acquire(&stream->started)) { + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + if (!clock_ready) { + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + if (!stream->clock_offset_valid || stream->timestamp_seed != update_seed) { + stream->clock_offset_ns = clock_offset_ns; + stream->timestamp_seed = update_seed; + stream->clock_offset_valid = true; + } + event_timestamp = ns_to_ktime((s64)dev_timestamp + stream->clock_offset_ns); + if (ktime_before(event_timestamp, stream->timestamp_accept_after)) { + pr_debug("t2bce_audio: ignoring pre-start timestamp dev=%s event=%lld accept_after=%lld\n", + sdev->uid, ktime_to_ns(event_timestamp), + ktime_to_ns(stream->timestamp_accept_after)); + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + stream->remote_timestamp = event_timestamp; + if (stream->waiting_for_first_ts) { + stream->waiting_for_first_ts = false; + snd_pcm_stream_unlock_irqrestore(substream, flags); + return; + } + snd_pcm_stream_unlock_irqrestore(substream, flags); + if (!substream->runtime->no_period_wakeup) + snd_pcm_period_elapsed(substream); +} + +void t2audio_handle_timestamp(struct t2audio_subdevice *sdev, u64 dev_timestamp, + u64 update_seed, s64 clock_offset_ns, bool clock_ready) +{ + struct snd_pcm_substream *substream; + + substream = sdev->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + if (substream) + t2audio_handle_stream_timestamp(substream, dev_timestamp, update_seed, + clock_offset_ns, clock_ready); + substream = sdev->pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream; + if (substream) + t2audio_handle_stream_timestamp(substream, dev_timestamp, update_seed, + clock_offset_ns, clock_ready); +} diff --git a/drivers/staging/t2bce/t2bce_audio/pcm.h b/drivers/staging/t2bce/t2bce_audio/pcm.h new file mode 100644 index 000000000..d39e63dd4 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/pcm.h @@ -0,0 +1,17 @@ +#ifndef T2AUDIO_PCM_H +#define T2AUDIO_PCM_H + +#include +#include + +struct t2audio_subdevice; +struct t2audio_apple_description; +struct snd_pcm_hardware; + +int t2audio_create_hw_info(struct t2audio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, size_t buf_size); +int t2audio_create_pcm(struct t2audio_subdevice *sdev); + +void t2audio_handle_timestamp(struct t2audio_subdevice *sdev, u64 dev_timestamp, + u64 update_seed, s64 clock_offset_ns, bool clock_ready); + +#endif //T2AUDIO_PCM_H diff --git a/drivers/staging/t2bce/t2bce_audio/protocol.c b/drivers/staging/t2bce/t2bce_audio/protocol.c new file mode 100644 index 000000000..7441bbb9c --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/protocol.c @@ -0,0 +1,347 @@ +#include "protocol.h" +#include "protocol_bce.h" +#include "audio.h" + +int t2audio_msg_read_base(struct t2audio_msg *msg, struct t2audio_msg_base *base) +{ + if (msg->size < sizeof(struct t2audio_msg_header) + sizeof(struct t2audio_msg_base) * 2) + return -EINVAL; + *base = *((struct t2audio_msg_base *) ((struct t2audio_msg_header *) msg->data + 1)); + return 0; +} + +#define READ_START(type) \ + size_t offset = sizeof(struct t2audio_msg_header) + sizeof(struct t2audio_msg_base); (void)offset; \ + if (((struct t2audio_msg_base *) ((struct t2audio_msg_header *) msg->data + 1))->msg != type) \ + return -EINVAL; +#define READ_DEVID_VAR(devid) *devid = ((struct t2audio_msg_header *) msg->data)->device_id +#define READ_VAL(type) ({ offset += sizeof(type); *((type *) ((u8 *) msg->data + offset - sizeof(type))); }) +#define READ_VAR(type, var) *var = READ_VAL(type) + +int t2audio_msg_read_start_io_response(struct t2audio_msg *msg) +{ + READ_START(T2AUDIO_MSG_START_IO_RESPONSE); + return 0; +} + +int t2audio_msg_read_stop_io_response(struct t2audio_msg *msg) +{ + READ_START(T2AUDIO_MSG_STOP_IO_RESPONSE); + return 0; +} + +int t2audio_msg_read_update_timestamp(struct t2audio_msg *msg, t2audio_device_id_t *devid, + u64 *timestamp, u64 *update_seed) +{ + READ_START(T2AUDIO_MSG_UPDATE_TIMESTAMP); + READ_DEVID_VAR(devid); + READ_VAR(u64, timestamp); + READ_VAR(u64, update_seed); + return 0; +} + +int t2audio_msg_read_get_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop, void **data, u64 *data_size) +{ + READ_START(T2AUDIO_MSG_GET_PROPERTY_RESPONSE); + READ_VAR(t2audio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + READ_VAR(u64, data_size); + *data = ((u8 *) msg->data + offset); + /* offset += data_size; */ + return 0; +} + +int t2audio_msg_read_set_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj) +{ + READ_START(T2AUDIO_MSG_SET_PROPERTY_RESPONSE); + READ_VAR(t2audio_object_id_t, obj); + return 0; +} + +int t2audio_msg_read_property_listener_response(struct t2audio_msg *msg, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop) +{ + READ_START(T2AUDIO_MSG_PROPERTY_LISTENER_RESPONSE); + READ_VAR(t2audio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + return 0; +} + +int t2audio_msg_read_property_changed(struct t2audio_msg *msg, t2audio_device_id_t *devid, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop) +{ + READ_START(T2AUDIO_MSG_PROPERTY_CHANGED); + READ_DEVID_VAR(devid); + READ_VAR(t2audio_object_id_t, obj); + READ_VAR(u32, &prop->element); + READ_VAR(u32, &prop->scope); + READ_VAR(u32, &prop->selector); + return 0; +} + +int t2audio_msg_read_set_input_stream_address_ranges_response(struct t2audio_msg *msg) +{ + READ_START(T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE); + return 0; +} + +int t2audio_msg_read_get_input_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt) +{ + READ_START(T2AUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE); + READ_VAR(u64, str_cnt); + *str_l = (t2audio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += str_cnt * sizeof(t2audio_object_id_t); */ + return 0; +} + +int t2audio_msg_read_get_output_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt) +{ + READ_START(T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE); + READ_VAR(u64, str_cnt); + *str_l = (t2audio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += str_cnt * sizeof(t2audio_object_id_t); */ + return 0; +} + +int t2audio_msg_read_set_remote_access_response(struct t2audio_msg *msg) +{ + READ_START(T2AUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE); + return 0; +} + +int t2audio_msg_read_get_device_list_response(struct t2audio_msg *msg, t2audio_device_id_t **dev_l, u64 *dev_cnt) +{ + READ_START(T2AUDIO_MSG_GET_DEVICE_LIST_RESPONSE); + READ_VAR(u64, dev_cnt); + *dev_l = (t2audio_device_id_t *) ((u8 *) msg->data + offset); + /* offset += dev_cnt * sizeof(t2audio_device_id_t); */ + return 0; +} + +#define WRITE_START_OF_TYPE(typev, devid) \ + size_t offset = sizeof(struct t2audio_msg_header); (void) offset; \ + ((struct t2audio_msg_header *) msg->data)->type = (typev); \ + ((struct t2audio_msg_header *) msg->data)->device_id = (devid); +#define WRITE_START_COMMAND(devid) WRITE_START_OF_TYPE(T2AUDIO_MSG_TYPE_COMMAND, devid) +#define WRITE_START_RESPONSE() WRITE_START_OF_TYPE(T2AUDIO_MSG_TYPE_RESPONSE, 0) +#define WRITE_START_NOTIFICATION() WRITE_START_OF_TYPE(T2AUDIO_MSG_TYPE_NOTIFICATION, 0) +#define WRITE_VAL(type, value) { *((type *) ((u8 *) msg->data + offset)) = value; offset += sizeof(value); } +#define WRITE_BIN(value, size) { memcpy((u8 *) msg->data + offset, value, size); offset += size; } +#define WRITE_BASE(type) WRITE_VAL(u32, type) WRITE_VAL(u32, 0) +#define WRITE_END() { msg->size = offset; } + +void t2audio_msg_write_start_io(struct t2audio_msg *msg, t2audio_device_id_t dev) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_START_IO); + WRITE_END(); +} + +void t2audio_msg_write_stop_io(struct t2audio_msg *msg, t2audio_device_id_t dev) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_STOP_IO); + WRITE_END(); +} + +void t2audio_msg_write_get_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_GET_PROPERTY); + WRITE_VAL(t2audio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_VAL(u64, qualifier_size); + WRITE_BIN(qualifier, qualifier_size); + WRITE_END(); +} + +void t2audio_msg_write_set_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_SET_PROPERTY); + WRITE_VAL(t2audio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_VAL(u64, data_size); + WRITE_BIN(data, data_size); + WRITE_VAL(u64, qualifier_size); + WRITE_BIN(qualifier, qualifier_size); + WRITE_END(); +} + +void t2audio_msg_write_property_listener(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop) +{ + WRITE_START_COMMAND(dev); + WRITE_BASE(T2AUDIO_MSG_PROPERTY_LISTENER); + WRITE_VAL(t2audio_object_id_t, obj); + WRITE_VAL(u32, prop.element); + WRITE_VAL(u32, prop.scope); + WRITE_VAL(u32, prop.selector); + WRITE_END(); +} + +void t2audio_msg_write_set_input_stream_address_ranges(struct t2audio_msg *msg, t2audio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES); + WRITE_END(); +} + +void t2audio_msg_write_get_input_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(T2AUDIO_MSG_GET_INPUT_STREAM_LIST); + WRITE_END(); +} + +void t2audio_msg_write_get_output_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid) +{ + WRITE_START_COMMAND(devid); + WRITE_BASE(T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST); + WRITE_END(); +} + +void t2audio_msg_write_set_remote_access(struct t2audio_msg *msg, u64 mode) +{ + WRITE_START_COMMAND(0); + WRITE_BASE(T2AUDIO_MSG_SET_REMOTE_ACCESS); + WRITE_VAL(u64, mode); + WRITE_END(); +} + +void t2audio_msg_write_alive_notification(struct t2audio_msg *msg, u32 proto_ver, u32 msg_ver) +{ + WRITE_START_NOTIFICATION(); + WRITE_BASE(T2AUDIO_MSG_NOTIFICATION_ALIVE); + WRITE_VAL(u32, proto_ver); + WRITE_VAL(u32, msg_ver); + WRITE_END(); +} + +void t2audio_msg_write_update_timestamp_response(struct t2audio_msg *msg) +{ + WRITE_START_RESPONSE(); + WRITE_BASE(T2AUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE); + WRITE_END(); +} + +void t2audio_msg_write_get_device_list(struct t2audio_msg *msg) +{ + WRITE_START_COMMAND(0); + WRITE_BASE(T2AUDIO_MSG_GET_DEVICE_LIST); + WRITE_END(); +} + +#define CMD_SHARED_VARS_NO_REPLY \ + int status = 0; \ + struct t2audio_send_ctx sctx; +#define CMD_SHARED_VARS \ + CMD_SHARED_VARS_NO_REPLY \ + struct t2audio_msg reply = t2audio_reply_alloc(); \ + struct t2audio_msg *buf = &reply; +#define CMD_SEND_REQUEST(fn, ...) \ + if ((status = t2audio_send_cmd_sync(a, &sctx, buf, 500, fn, ##__VA_ARGS__))) \ + return status; +#define CMD_DEF_SHARED_AND_SEND(fn, ...) \ + CMD_SHARED_VARS \ + CMD_SEND_REQUEST(fn, ##__VA_ARGS__); +#define CMD_DEF_SHARED_NO_REPLY_AND_SEND(fn, ...) \ + CMD_SHARED_VARS_NO_REPLY \ + CMD_SEND_REQUEST(fn, ##__VA_ARGS__); +#define CMD_HNDL_REPLY_NO_FREE(fn, ...) \ + status = fn(buf, ##__VA_ARGS__); \ + return status; +#define CMD_HNDL_REPLY_AND_FREE(fn, ...) \ + status = fn(buf, ##__VA_ARGS__); \ + t2audio_reply_free(&reply); \ + return status; + +int t2audio_cmd_start_io(struct t2audio_device *a, t2audio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_start_io, devid); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_start_io_response); +} +int t2audio_cmd_stop_io(struct t2audio_device *a, t2audio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_stop_io, devid); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_stop_io_response); +} +int t2audio_cmd_get_property(struct t2audio_device *a, struct t2audio_msg *buf, + t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_property, devid, obj, prop, qualifier, qualifier_size); + CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_property_response, &obj, &prop, data, data_size); +} +int t2audio_cmd_get_primitive_property(struct t2audio_device *a, + t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) +{ + int status; + struct t2audio_msg reply = t2audio_reply_alloc(); + void *r_data; + u64 r_data_size; + if ((status = t2audio_cmd_get_property(a, &reply, devid, obj, prop, qualifier, qualifier_size, + &r_data, &r_data_size))) + goto finish; + if (r_data_size != data_size) { + status = -EINVAL; + goto finish; + } + memcpy(data, r_data, data_size); +finish: + t2audio_reply_free(&reply); + return status; +} +int t2audio_cmd_set_property(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_set_property, devid, obj, prop, data, data_size, + qualifier, qualifier_size); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_set_property_response, &obj); +} +int t2audio_cmd_property_listener(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_property_listener, devid, obj, prop); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_property_listener_response, &obj, &prop); +} +int t2audio_cmd_set_input_stream_address_ranges(struct t2audio_device *a, t2audio_device_id_t devid) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_set_input_stream_address_ranges, devid); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_set_input_stream_address_ranges_response); +} +int t2audio_cmd_get_input_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid, + t2audio_object_id_t **str_l, u64 *str_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_input_stream_list, devid); + CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_input_stream_list_response, str_l, str_cnt); +} +int t2audio_cmd_get_output_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid, + t2audio_object_id_t **str_l, u64 *str_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_output_stream_list, devid); + CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_output_stream_list_response, str_l, str_cnt); +} +int t2audio_cmd_set_remote_access(struct t2audio_device *a, u64 mode) +{ + CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_set_remote_access, mode); + CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_set_remote_access_response); +} +int t2audio_cmd_get_device_list(struct t2audio_device *a, struct t2audio_msg *buf, + t2audio_device_id_t **dev_l, u64 *dev_cnt) +{ + CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_device_list); + CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_device_list_response, dev_l, dev_cnt); +} \ No newline at end of file diff --git a/drivers/staging/t2bce/t2bce_audio/protocol.h b/drivers/staging/t2bce/t2bce_audio/protocol.h new file mode 100644 index 000000000..650d49793 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/protocol.h @@ -0,0 +1,147 @@ +#ifndef T2AUDIO_PROTOCOL_H +#define T2AUDIO_PROTOCOL_H + +#include + +struct t2audio_device; + +typedef u64 t2audio_device_id_t; +typedef u64 t2audio_object_id_t; + +struct t2audio_msg { + void *data; + size_t size; +}; + +struct __attribute__((packed)) t2audio_msg_header { + char tag[4]; + u8 type; + t2audio_device_id_t device_id; +}; +struct __attribute__((packed)) t2audio_msg_base { + u32 msg; + u32 status; +}; + +struct t2audio_prop_addr { + u32 scope; + u32 selector; + u32 element; +}; +#define T2AUDIO_PROP(scope, sel, el) (struct t2audio_prop_addr) { scope, sel, el } + +enum { + T2AUDIO_MSG_TYPE_COMMAND = 1, + T2AUDIO_MSG_TYPE_RESPONSE = 2, + T2AUDIO_MSG_TYPE_NOTIFICATION = 3 +}; + +enum { + T2AUDIO_MSG_START_IO = 0, + T2AUDIO_MSG_START_IO_RESPONSE = 1, + T2AUDIO_MSG_STOP_IO = 2, + T2AUDIO_MSG_STOP_IO_RESPONSE = 3, + T2AUDIO_MSG_UPDATE_TIMESTAMP = 4, + T2AUDIO_MSG_GET_PROPERTY = 7, + T2AUDIO_MSG_GET_PROPERTY_RESPONSE = 8, + T2AUDIO_MSG_SET_PROPERTY = 9, + T2AUDIO_MSG_SET_PROPERTY_RESPONSE = 10, + T2AUDIO_MSG_PROPERTY_LISTENER = 11, + T2AUDIO_MSG_PROPERTY_LISTENER_RESPONSE = 12, + T2AUDIO_MSG_PROPERTY_CHANGED = 13, + T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES = 18, + T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE = 19, + T2AUDIO_MSG_GET_INPUT_STREAM_LIST = 24, + T2AUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE = 25, + T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST = 26, + T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE = 27, + T2AUDIO_MSG_SET_REMOTE_ACCESS = 32, + T2AUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE = 33, + T2AUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE = 34, + + T2AUDIO_MSG_NOTIFICATION_ALIVE = 100, + T2AUDIO_MSG_GET_DEVICE_LIST = 101, + T2AUDIO_MSG_GET_DEVICE_LIST_RESPONSE = 102, + T2AUDIO_MSG_NOTIFICATION_BOOT = 104 +}; + +enum { + T2AUDIO_REMOTE_ACCESS_OFF = 0, + T2AUDIO_REMOTE_ACCESS_ON = 2 +}; + +enum { + T2AUDIO_PROP_SCOPE_GLOBAL = 0x676c6f62, // 'glob' + T2AUDIO_PROP_SCOPE_INPUT = 0x696e7074, // 'inpt' + T2AUDIO_PROP_SCOPE_OUTPUT = 0x6f757470 // 'outp' +}; + +enum { + T2AUDIO_PROP_UID = 0x75696420, // 'uid ' + T2AUDIO_PROP_BOOL_VALUE = 0x6263766c, // 'bcvl' + T2AUDIO_PROP_JACK_PLUGGED = 0x6a61636b, // 'jack' + T2AUDIO_PROP_SEL_VOLUME = 0x64656176, // 'deav' + T2AUDIO_PROP_LATENCY = 0x6c746e63, // 'ltnc' + T2AUDIO_PROP_PHYS_FORMAT = 0x70667420 // 'pft ' +}; + +int t2audio_msg_read_base(struct t2audio_msg *msg, struct t2audio_msg_base *base); + +int t2audio_msg_read_start_io_response(struct t2audio_msg *msg); +int t2audio_msg_read_stop_io_response(struct t2audio_msg *msg); +int t2audio_msg_read_update_timestamp(struct t2audio_msg *msg, t2audio_device_id_t *devid, + u64 *timestamp, u64 *update_seed); +int t2audio_msg_read_get_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop, void **data, u64 *data_size); +int t2audio_msg_read_set_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj); +int t2audio_msg_read_property_listener_response(struct t2audio_msg *msg,t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop); +int t2audio_msg_read_property_changed(struct t2audio_msg *msg, t2audio_device_id_t *devid, t2audio_object_id_t *obj, + struct t2audio_prop_addr *prop); +int t2audio_msg_read_set_input_stream_address_ranges_response(struct t2audio_msg *msg); +int t2audio_msg_read_get_input_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt); +int t2audio_msg_read_get_output_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt); +int t2audio_msg_read_set_remote_access_response(struct t2audio_msg *msg); +int t2audio_msg_read_get_device_list_response(struct t2audio_msg *msg, t2audio_device_id_t **dev_l, u64 *dev_cnt); + +void t2audio_msg_write_start_io(struct t2audio_msg *msg, t2audio_device_id_t dev); +void t2audio_msg_write_stop_io(struct t2audio_msg *msg, t2audio_device_id_t dev); +void t2audio_msg_write_get_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size); +void t2audio_msg_write_set_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size); +void t2audio_msg_write_property_listener(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj, + struct t2audio_prop_addr prop); +void t2audio_msg_write_set_input_stream_address_ranges(struct t2audio_msg *msg, t2audio_device_id_t devid); +void t2audio_msg_write_get_input_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid); +void t2audio_msg_write_get_output_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid); +void t2audio_msg_write_set_remote_access(struct t2audio_msg *msg, u64 mode); +void t2audio_msg_write_alive_notification(struct t2audio_msg *msg, u32 proto_ver, u32 msg_ver); +void t2audio_msg_write_update_timestamp_response(struct t2audio_msg *msg); +void t2audio_msg_write_get_device_list(struct t2audio_msg *msg); + + +int t2audio_cmd_start_io(struct t2audio_device *a, t2audio_device_id_t devid); +int t2audio_cmd_stop_io(struct t2audio_device *a, t2audio_device_id_t devid); +int t2audio_cmd_get_property(struct t2audio_device *a, struct t2audio_msg *buf, + t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size); +int t2audio_cmd_get_primitive_property(struct t2audio_device *a, + t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); +int t2audio_cmd_set_property(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size); +int t2audio_cmd_property_listener(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj, + struct t2audio_prop_addr prop); +int t2audio_cmd_set_input_stream_address_ranges(struct t2audio_device *a, t2audio_device_id_t devid); +int t2audio_cmd_get_input_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid, + t2audio_object_id_t **str_l, u64 *str_cnt); +int t2audio_cmd_get_output_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid, + t2audio_object_id_t **str_l, u64 *str_cnt); +int t2audio_cmd_set_remote_access(struct t2audio_device *a, u64 mode); +int t2audio_cmd_get_device_list(struct t2audio_device *a, struct t2audio_msg *buf, + t2audio_device_id_t **dev_l, u64 *dev_cnt); + + + +#endif //T2AUDIO_PROTOCOL_H diff --git a/drivers/staging/t2bce/t2bce_audio/protocol_bce.c b/drivers/staging/t2bce/t2bce_audio/protocol_bce.c new file mode 100644 index 000000000..a6f3de971 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/protocol_bce.c @@ -0,0 +1,260 @@ +#include "protocol_bce.h" + +#include "audio.h" +#include + +static void t2audio_bce_out_queue_completion(struct t2bce_core_queue_sq *sq); +static void t2audio_bce_in_queue_completion(struct t2bce_core_queue_sq *sq); +static int t2audio_bce_queue_init(struct t2audio_device *dev, struct t2audio_bce_queue *q, const char *name, + enum dma_data_direction direction, t2bce_core_sq_completion cfn); +void t2audio_bce_in_queue_submit_pending(struct t2audio_bce_queue *q, size_t count); +static void t2audio_deferred_msg_work(struct work_struct *ws); + +int t2audio_bce_init(struct t2audio_device *dev) +{ + int status; + struct t2audio_bce *bce = &dev->bcem; + bce->cq = t2bce_core_create_cq(dev->bce, 0x80); + spin_lock_init(&bce->spinlock); + if (!bce->cq) + return -EINVAL; + if ((status = t2audio_bce_queue_init(dev, &bce->qout, "com.apple.BridgeAudio.IntelToARM", DMA_TO_DEVICE, + t2audio_bce_out_queue_completion))) { + return status; + } + if ((status = t2audio_bce_queue_init(dev, &bce->qin, "com.apple.BridgeAudio.ARMToIntel", DMA_FROM_DEVICE, + t2audio_bce_in_queue_completion))) { + return status; + } + t2audio_bce_in_queue_submit_pending(&bce->qin, bce->qin.el_count); + return 0; +} + +int t2audio_bce_queue_init(struct t2audio_device *dev, struct t2audio_bce_queue *q, const char *name, + enum dma_data_direction direction, t2bce_core_sq_completion cfn) +{ + q->cq = dev->bcem.cq; + q->el_size = T2AUDIO_BCE_QUEUE_ELEMENT_SIZE; + q->el_count = T2AUDIO_BCE_QUEUE_ELEMENT_COUNT; + /* Allocate one extra SQ slot because the BCE ring always keeps one entry empty. */ + q->sq = t2bce_core_create_sq(dev->bce, q->cq, name, (u32) (q->el_count + 1), direction, cfn, dev); + if (!q->sq) + return -EINVAL; + + q->data = dma_alloc_coherent(t2bce_core_client_dma_dev(dev->bce), q->el_size * q->el_count, &q->dma_addr, GFP_KERNEL); + if (!q->data) { + t2bce_core_destroy_sq(dev->bce, q->sq); + return -EINVAL; + } + return 0; +} + +static void t2audio_send_create_tag(struct t2audio_bce *b, int *tagn, char tag[4]) +{ + b->tag_num = (b->tag_num + 1) % T2AUDIO_BCE_QUEUE_TAG_COUNT; + *tagn = b->tag_num; + tag[0] = 'S'; + tag[1] = '0' + b->tag_num / 100; + tag[2] = '0' + (b->tag_num / 10) % 10; + tag[3] = '0' + b->tag_num % 10; +} + +int __t2audio_send_prepare(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, char *tag) +{ + int status; + size_t index; + void *dptr; + struct t2audio_msg_header *header; + if ((status = t2bce_core_reserve_submission(b->qout.sq, &ctx->timeout))) + return status; + spin_lock_irqsave(&b->spinlock, ctx->irq_flags); + index = b->qout.data_tail; + dptr = (u8 *) b->qout.data + index * b->qout.el_size; + ctx->msg.data = dptr; + header = dptr; + if (tag) + *((u32 *) header->tag) = *((u32 *) tag); + else + t2audio_send_create_tag(b, &ctx->tag_n, header->tag); + return 0; +} + +void __t2audio_send(struct t2audio_bce *b, struct t2audio_send_ctx *ctx) +{ +#ifdef DEBUG + pr_debug("t2bce_audio: Sending command data\n"); + print_hex_dump(KERN_DEBUG, "t2bce_audio:OUT ", DUMP_PREFIX_NONE, 32, 1, ctx->msg.data, ctx->msg.size, true); +#endif + t2bce_core_set_next_submission_single(b->qout.sq, + b->qout.dma_addr + (dma_addr_t) (ctx->msg.data - b->qout.data), ctx->msg.size); + t2bce_core_submit_to_device(b->qout.sq); + b->qout.data_tail = (b->qout.data_tail + 1) % b->qout.el_count; + spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); +} + +int __t2audio_send_cmd_sync(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, struct t2audio_msg *reply) +{ + struct t2audio_bce_queue_entry ent; + DECLARE_COMPLETION_ONSTACK(cmpl); + ent.msg = reply; + ent.cmpl = &cmpl; + b->pending_entries[ctx->tag_n] = &ent; + __t2audio_send(b, ctx); /* drops b->spinlock */ + ctx->timeout = wait_for_completion_timeout(&cmpl, ctx->timeout); + if (ctx->timeout == 0) { + /* + * Timed-out replies may still arrive later; remove the waiter before + * returning so a stale reply cannot complete a reused stack object. + */ + spin_lock_irqsave(&b->spinlock, ctx->irq_flags); + if (b->pending_entries[ctx->tag_n] == &ent) + b->pending_entries[ctx->tag_n] = NULL; + spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags); + return -ETIMEDOUT; + } + return 0; +} + +static void t2audio_handle_reply(struct t2audio_bce *b, struct t2audio_msg *reply) +{ + const char *tag; + int tagn; + unsigned long irq_flags; + char tag_zero[5]; + struct t2audio_bce_queue_entry *entry; + + tag = ((struct t2audio_msg_header *) reply->data)->tag; + if (tag[0] != 'S') { + pr_err("t2bce_audio: unexpected reply tag: %.4s\n", tag); + return; + } + *((u32 *) tag_zero) = *((u32 *) tag); + tag_zero[4] = 0; + if (kstrtoint(&tag_zero[1], 10, &tagn)) { + pr_err("t2bce_audio: reply tag parse failed: %.4s\n", tag); + return; + } + + spin_lock_irqsave(&b->spinlock, irq_flags); + entry = b->pending_entries[tagn]; + if (entry) { + if (reply->size < entry->msg->size) + entry->msg->size = reply->size; + memcpy(entry->msg->data, reply->data, entry->msg->size); + complete(entry->cmpl); + + b->pending_entries[tagn] = NULL; + } else { + pr_warn_ratelimited("t2bce_audio: stale reply without queued command tag=%.4s\n", tag); + } + spin_unlock_irqrestore(&b->spinlock, irq_flags); +} + +static void t2audio_bce_out_queue_completion(struct t2bce_core_queue_sq *sq) +{ + while (t2bce_core_next_completion(sq)) { + t2bce_core_notify_submission_complete(sq); + } +} + +static void t2audio_bce_in_queue_handle_msg(struct t2audio_device *a, struct t2audio_msg *msg); + +static void t2audio_deferred_msg_work(struct work_struct *ws) +{ + struct t2audio_deferred_msg *work = container_of(ws, struct t2audio_deferred_msg, ws); + struct t2audio_msg_header *header = work->msg.data; + + if (header->type == T2AUDIO_MSG_TYPE_COMMAND) + t2audio_handle_command(work->a, &work->msg); + else if (header->type == T2AUDIO_MSG_TYPE_NOTIFICATION) + t2audio_handle_notification(work->a, &work->msg); + + kfree(work->msg.data); + kfree(work); +} + +static void t2audio_bce_in_queue_completion(struct t2bce_core_queue_sq *sq) +{ + struct t2audio_msg msg; + struct t2audio_device *dev = t2bce_core_queue_sq_userdata(sq); + struct t2audio_bce_queue *q = &dev->bcem.qin; + struct t2bce_core_sq_completion_data *c; + size_t cnt = 0; + + mb(); + while ((c = t2bce_core_next_completion(sq))) { + msg.data = (u8 *) q->data + q->data_head * q->el_size; + msg.size = c->data_size; +#ifdef DEBUG + pr_debug("t2bce_audio: Received command data %llx\n", c->data_size); + print_hex_dump(KERN_DEBUG, "t2bce_audio:IN ", DUMP_PREFIX_NONE, 32, 1, msg.data, min(msg.size, 128UL), true); +#endif + t2audio_bce_in_queue_handle_msg(dev, &msg); + + q->data_head = (q->data_head + 1) % q->el_count; + + t2bce_core_notify_submission_complete(sq); + ++cnt; + } + t2audio_bce_in_queue_submit_pending(q, cnt); +} + +static void t2audio_bce_in_queue_handle_msg(struct t2audio_device *a, struct t2audio_msg *msg) +{ + struct t2audio_msg_header *header = (struct t2audio_msg_header *) msg->data; + struct t2audio_deferred_msg *work; + + if (msg->size < sizeof(struct t2audio_msg_header)) { + pr_err("t2bce_audio: Msg size smaller than header (%lx)", msg->size); + return; + } + if (header->type == T2AUDIO_MSG_TYPE_RESPONSE) { + t2audio_handle_reply(&a->bcem, msg); + return; + } + + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) { + pr_err("t2bce_audio: Failed to allocate deferred message work\n"); + return; + } + + work->msg.data = kmemdup(msg->data, msg->size, GFP_ATOMIC); + if (!work->msg.data) { + kfree(work); + pr_err("t2bce_audio: Failed to copy deferred message\n"); + return; + } + + work->a = a; + work->msg.size = msg->size; + INIT_WORK(&work->ws, t2audio_deferred_msg_work); + schedule_work(&work->ws); +} + +void t2audio_bce_in_queue_submit_pending(struct t2audio_bce_queue *q, size_t count) +{ + while (count--) { + if (t2bce_core_reserve_submission(q->sq, NULL)) { + pr_err("t2bce_audio: Failed to reserve an event queue submission\n"); + break; + } + t2bce_core_set_next_submission_single(q->sq, + q->dma_addr + (dma_addr_t) (q->data_tail * q->el_size), q->el_size); + q->data_tail = (q->data_tail + 1) % q->el_count; + } + t2bce_core_submit_to_device(q->sq); +} + +struct t2audio_msg t2audio_reply_alloc(void) +{ + struct t2audio_msg ret; + ret.size = T2AUDIO_BCE_QUEUE_ELEMENT_SIZE; + ret.data = kmalloc(ret.size, GFP_KERNEL); + return ret; +} + +void t2audio_reply_free(struct t2audio_msg *reply) +{ + kfree(reply->data); +} diff --git a/drivers/staging/t2bce/t2bce_audio/protocol_bce.h b/drivers/staging/t2bce/t2bce_audio/protocol_bce.h new file mode 100644 index 000000000..8e474d2f7 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_audio/protocol_bce.h @@ -0,0 +1,72 @@ +#ifndef T2AUDIO_PROTOCOL_BCE_H +#define T2AUDIO_PROTOCOL_BCE_H + +#include "protocol.h" +#include "t2bce_core_transport.h" + +#define T2AUDIO_BCE_QUEUE_ELEMENT_SIZE 0x1000 +#define T2AUDIO_BCE_QUEUE_ELEMENT_COUNT 20 + +#define T2AUDIO_BCE_QUEUE_TAG_COUNT 1000 + +struct t2audio_device; + +struct t2audio_bce_queue_entry { + struct t2audio_msg *msg; + struct completion *cmpl; +}; +struct t2audio_bce_queue { + struct t2bce_core_queue_cq *cq; + struct t2bce_core_queue_sq *sq; + void *data; + dma_addr_t dma_addr; + size_t data_head, data_tail; + size_t el_size, el_count; +}; +struct t2audio_bce { + struct t2bce_core_queue_cq *cq; + struct t2audio_bce_queue qin; + struct t2audio_bce_queue qout; + int tag_num; + struct t2audio_bce_queue_entry *pending_entries[T2AUDIO_BCE_QUEUE_TAG_COUNT]; + struct spinlock spinlock; +}; + +struct t2audio_send_ctx { + int status; + int tag_n; + unsigned long irq_flags; + struct t2audio_msg msg; + unsigned long timeout; +}; + +int t2audio_bce_init(struct t2audio_device *dev); +int __t2audio_send_prepare(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, char *tag); +void __t2audio_send(struct t2audio_bce *b, struct t2audio_send_ctx *ctx); +int __t2audio_send_cmd_sync(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, struct t2audio_msg *reply); + +#define t2audio_send_with_tag(a, ctx, tag, tout, fn, ...) ({ \ + (ctx)->timeout = msecs_to_jiffies(tout); \ + (ctx)->status = __t2audio_send_prepare(&(a)->bcem, (ctx), (tag)); \ + if (!(ctx)->status) { \ + fn(&(ctx)->msg, ##__VA_ARGS__); \ + __t2audio_send(&(a)->bcem, (ctx)); \ + } \ + (ctx)->status; \ +}) +#define t2audio_send(a, ctx, tout, fn, ...) t2audio_send_with_tag(a, ctx, NULL, tout, fn, ##__VA_ARGS__) + +#define t2audio_send_cmd_sync(a, ctx, reply, tout, fn, ...) ({ \ + (ctx)->timeout = msecs_to_jiffies(tout); \ + (ctx)->status = __t2audio_send_prepare(&(a)->bcem, (ctx), NULL); \ + if (!(ctx)->status) { \ + fn(&(ctx)->msg, ##__VA_ARGS__); \ + (ctx)->status = __t2audio_send_cmd_sync(&(a)->bcem, (ctx), (reply)); \ + } \ + (ctx)->status; \ +}) + +struct t2audio_msg t2audio_reply_alloc(void); +void t2audio_reply_free(struct t2audio_msg *reply); + +#endif //T2AUDIO_PROTOCOL_BCE_H diff --git a/drivers/staging/t2bce/t2bce_core/Makefile b/drivers/staging/t2bce/t2bce_core/Makefile new file mode 100644 index 000000000..5ee2110b8 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_core/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_CORE) += t2bce_core.o +t2bce_core-y := t2bce_main.o mailbox.o transport.o + +ccflags-y += -DWITHOUT_NVME_PATCH +ccflags-y += -I$(src)/include +ccflags-y += -I$(src)/../t2bce_dma/include diff --git a/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h b/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h new file mode 100644 index 000000000..c0085c266 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h @@ -0,0 +1,83 @@ +#ifndef T2BCE_CORE_TRANSPORT_H +#define T2BCE_CORE_TRANSPORT_H + +#include +#include +#include + +struct t2bce_core_client; +struct t2bce_core_queue_cq; +struct t2bce_core_queue_sq; + +typedef void (*t2bce_core_sq_completion)(struct t2bce_core_queue_sq *sq); +typedef void (*t2bce_core_client_resume_callback)(void *userdata); + +struct t2bce_core_client_pm_ops { + void (*shutdown)(void *userdata); + void (*pm_reset)(void *userdata); + void (*pm_prepare)(void *userdata); + void (*pm_prepare_no_state)(void *userdata); + void (*pm_mark_no_state_resume)(void *userdata); + bool (*pm_is_no_state_resume)(void *userdata); + void (*pm_complete)(void *userdata); +}; + +struct t2bce_core_sq_completion_data { + u32 status; + u64 data_size; + u64 result; +}; + +struct t2bce_core_dma_segment { + u64 addr; + u64 length; +}; + +enum t2bce_completion_status { + T2BCE_COMPLETION_SUCCESS = 0, + T2BCE_COMPLETION_ERROR = 1, + T2BCE_COMPLETION_ABORTED = 2, + T2BCE_COMPLETION_NO_SPACE = 3, + T2BCE_COMPLETION_OVERRUN = 4, +}; + +struct t2bce_core_client *t2bce_core_client_get(struct device *dev); +void t2bce_core_client_put(struct t2bce_core_client *client); + +struct device *t2bce_core_device_get(void); +void t2bce_core_device_put(struct device *dev); +struct device *t2bce_core_client_dma_dev(struct t2bce_core_client *client); +bool t2bce_core_client_no_state_resume(struct t2bce_core_client *client); + +void t2bce_core_client_set_resume_complete_callback(struct t2bce_core_client *client, + t2bce_core_client_resume_callback callback, void *userdata); +void t2bce_core_client_set_pm_ops(struct t2bce_core_client *client, + const struct t2bce_core_client_pm_ops *ops, void *userdata); + +struct t2bce_core_queue_cq *t2bce_core_create_cq(struct t2bce_core_client *client, u32 el_count); +void t2bce_core_destroy_cq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq); + +struct t2bce_core_queue_sq *t2bce_core_create_sq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq, + const char *name, u32 el_count, enum dma_data_direction direction, + t2bce_core_sq_completion compl, void *userdata); +void t2bce_core_destroy_sq(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq); + +void *t2bce_core_queue_sq_userdata(struct t2bce_core_queue_sq *sq); + +int t2bce_core_reserve_submission(struct t2bce_core_queue_sq *sq, unsigned long *timeout); +void t2bce_core_cancel_submission_reservation(struct t2bce_core_queue_sq *sq); +void t2bce_core_set_next_submission_single(struct t2bce_core_queue_sq *sq, dma_addr_t addr, size_t size); +void t2bce_core_set_next_submission_segment_list(struct t2bce_core_queue_sq *sq, + dma_addr_t segl_addr, size_t segl_size); +void t2bce_core_submit_to_device(struct t2bce_core_queue_sq *sq); +void t2bce_core_notify_submission_complete(struct t2bce_core_queue_sq *sq); + +struct t2bce_core_sq_completion_data *t2bce_core_next_completion(struct t2bce_core_queue_sq *sq); + +u32 t2bce_core_queue_sq_head(struct t2bce_core_queue_sq *sq); +u32 t2bce_core_queue_sq_tail(struct t2bce_core_queue_sq *sq); +u32 t2bce_core_queue_sq_available(struct t2bce_core_queue_sq *sq); +u32 t2bce_core_queue_sq_capacity(struct t2bce_core_queue_sq *sq); +int t2bce_core_flush_queue(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq); + +#endif diff --git a/drivers/staging/t2bce/t2bce_core/mailbox.c b/drivers/staging/t2bce/t2bce_core/mailbox.c new file mode 100644 index 000000000..45aa999f7 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_core/mailbox.c @@ -0,0 +1,234 @@ +#include "mailbox.h" +#include +#include "t2bce.h" +#include + +#define REG_MBOX_OUT_BASE 0x820 +#define REG_MBOX_REPLY_COUNTER 0x108 +#define REG_MBOX_REPLY_BASE 0x810 +#define REG_TIMESTAMP_BASE 0xC000 + +#define BCE_MBOX_TIMEOUT_MS 1000 + +void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb) +{ + mb->reg_mb = reg_mb; + mutex_init(&mb->channel_lock); + init_completion(&mb->mb_completion); +} + +int bce_mailbox_channel_pause(struct bce_mailbox *mb) +{ + /* + * System PM needs to exclude new mailbox transactions while the core + * driver is moving bridgeOS into or out of sleep. + */ + mutex_lock(&mb->channel_lock); + return 0; +} + +void bce_mailbox_channel_resume(struct bce_mailbox *mb) +{ + mutex_unlock(&mb->channel_lock); +} + +int bce_mailbox_send_locked_timeout(struct bce_mailbox *mb, u64 msg, u64* recv, unsigned int timeout_ms) +{ + u32 __iomem *regb; + int status = 0; + + if (atomic_cmpxchg(&mb->mb_status, 0, 1) != 0) { + status = -EEXIST; + goto out; + } + reinit_completion(&mb->mb_completion); + + pr_debug("bce_mailbox_send: %llx\n", msg); + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_OUT_BASE); + iowrite32((u32) msg, regb); + iowrite32((u32) (msg >> 32), regb + 1); + iowrite32(0, regb + 2); + iowrite32(0, regb + 3); + + wait_for_completion_timeout(&mb->mb_completion, msecs_to_jiffies(timeout_ms)); + if (atomic_read(&mb->mb_status) != 2) { + atomic_set(&mb->mb_status, 0); + status = -ETIMEDOUT; + goto out; + } + + *recv = mb->mb_result; + pr_debug("bce_mailbox_send: reply %llx\n", *recv); + + atomic_set(&mb->mb_status, 0); +out: + return status; +} + +int bce_mailbox_send_timeout(struct bce_mailbox *mb, u64 msg, u64* recv, unsigned int timeout_ms) +{ + int status; + + /* Serialize mailbox traffic; bridgeOS accepts one active command at a time. */ + mutex_lock(&mb->channel_lock); + status = bce_mailbox_send_locked_timeout(mb, msg, recv, timeout_ms); + mutex_unlock(&mb->channel_lock); + return status; +} + +int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv) +{ + return bce_mailbox_send_timeout(mb, msg, recv, BCE_MBOX_TIMEOUT_MS); +} + +int bce_mailbox_send_locked(struct bce_mailbox *mb, u64 msg, u64 *recv) +{ + return bce_mailbox_send_locked_timeout(mb, msg, recv, BCE_MBOX_TIMEOUT_MS); +} + +int bce_mailbox_send_no_reply_locked(struct bce_mailbox *mb, u64 msg) +{ + u32 __iomem *regb; + int status = 0; + + if (atomic_cmpxchg(&mb->mb_status, 0, 1) != 0) { + status = -EEXIST; + goto out; + } + + pr_debug("bce_mailbox_send_no_reply: %llx\n", msg); + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_OUT_BASE); + iowrite32((u32) msg, regb); + iowrite32((u32) (msg >> 32), regb + 1); + iowrite32(0, regb + 2); + iowrite32(0, regb + 3); + + atomic_set(&mb->mb_status, 0); +out: + return status; +} + +int bce_mailbox_send_no_reply(struct bce_mailbox *mb, u64 msg) +{ + int status; + + /* Fire-and-forget commands share the same mailbox serialization. */ + mutex_lock(&mb->channel_lock); + status = bce_mailbox_send_no_reply_locked(mb, msg); + mutex_unlock(&mb->channel_lock); + return status; +} + +static int bce_mailbox_retrive_response(struct bce_mailbox *mb) +{ + u32 __iomem *regb; + u32 lo, hi; + int count, counter; + u32 res = ioread32((u8*) mb->reg_mb + REG_MBOX_REPLY_COUNTER); + count = (res >> 20) & 0xf; + counter = count; + pr_debug("bce_mailbox_retrive_response count=%i\n", count); + while (counter--) { + regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_REPLY_BASE); + lo = ioread32(regb); + hi = ioread32(regb + 1); + ioread32(regb + 2); + ioread32(regb + 3); + pr_debug("bce_mailbox_retrive_response %llx\n", ((u64) hi << 32) | lo); + mb->mb_result = ((u64) hi << 32) | lo; + } + return count > 0 ? 0 : -ENODATA; +} + +int bce_mailbox_handle_interrupt(struct bce_mailbox *mb) +{ + int status = bce_mailbox_retrive_response(mb); + if (!status) { + if (atomic_read(&mb->mb_status) == 1) { + atomic_set(&mb->mb_status, 2); + complete(&mb->mb_completion); + } else { + pr_debug("bce_mailbox_handle_interrupt: unsolicited reply %llx\n", mb->mb_result); + atomic_set(&mb->mb_status, 0); + } + } + return status; +} + +static void bce_xhci_pm_tick(struct timer_list *tl); + +void bce_xhci_pm_init(struct bce_xhci_pm *pm, void __iomem *reg) +{ + u32 __iomem *regb; + + spin_lock_init(&pm->stop_sl); + pm->stopped = false; + + pm->reg = reg; + + regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE); + + ioread32(regb); + mb(); + + timer_setup(&pm->timer, bce_xhci_pm_tick, 0); +} + +void bce_xhci_pm_start(struct bce_xhci_pm *pm, bool is_initial) +{ + unsigned long flags; + u32 __iomem *regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE); + + if (is_initial) { + iowrite32((u32) -4, regb + 2); + iowrite32((u32) -1, regb); + } else { + iowrite32((u32) -3, regb + 2); + iowrite32((u32) -1, regb); + } + + spin_lock_irqsave(&pm->stop_sl, flags); + pm->stopped = false; + spin_unlock_irqrestore(&pm->stop_sl, flags); + mod_timer(&pm->timer, jiffies + msecs_to_jiffies(150)); +} + +void bce_xhci_pm_stop(struct bce_xhci_pm *pm) +{ + unsigned long flags; + u32 __iomem *regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE); + + spin_lock_irqsave(&pm->stop_sl, flags); + pm->stopped = true; + spin_unlock_irqrestore(&pm->stop_sl, flags); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,15,0) + del_timer_sync(&pm->timer); +#else + timer_delete_sync(&pm->timer); +#endif + iowrite32((u32) -2, regb + 2); + iowrite32((u32) -1, regb); +} + +static void bce_xhci_pm_tick(struct timer_list *tl) +{ + struct bce_xhci_pm *pm; + unsigned long flags; + u32 __iomem *regb; + ktime_t bt; + + pm = container_of(tl, struct bce_xhci_pm, timer); + regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE); + local_irq_save(flags); + ioread32(regb + 2); + mb(); + bt = ktime_get_boottime(); + iowrite32((u32) bt, regb + 2); + iowrite32((u32) (bt >> 32), regb); + + spin_lock(&pm->stop_sl); + if (!pm->stopped) + mod_timer(&pm->timer, jiffies + msecs_to_jiffies(150)); + spin_unlock(&pm->stop_sl); + local_irq_restore(flags); +} diff --git a/drivers/staging/t2bce/t2bce_core/mailbox.h b/drivers/staging/t2bce/t2bce_core/mailbox.h new file mode 100644 index 000000000..8a16bd36a --- /dev/null +++ b/drivers/staging/t2bce/t2bce_core/mailbox.h @@ -0,0 +1,64 @@ +#ifndef BCE_MAILBOX_H +#define BCE_MAILBOX_H + +#include +#include +#include +#include + +struct bce_mailbox { + void __iomem *reg_mb; + struct mutex channel_lock; + + atomic_t mb_status; // possible statuses: 0 (no msg), 1 (has active msg), 2 (got reply) + struct completion mb_completion; + uint64_t mb_result; +}; + +enum bce_message_type { + BCE_MB_REGISTER_COMMAND_SQ = 0x7, // to-device + BCE_MB_REGISTER_COMMAND_CQ = 0x8, // to-device + BCE_MB_REGISTER_COMMAND_QUEUE_REPLY = 0xB, // to-host + BCE_MB_SET_FW_PROTOCOL_VERSION = 0xC, // both + BCE_MB_SLEEP_NO_STATE = 0x14, // to-device + BCE_MB_RESTORE_NO_STATE = 0x15, // to-device + BCE_MB_SLEEP_LIGHT = 0x16, // to-device, unused; reserved for future runtime-PM work + BCE_MB_SAVE_STATE_AND_SLEEP = 0x17, // to-device + BCE_MB_RESTORE_STATE_AND_WAKE = 0x18, // to-device + BCE_MB_SAVE_STATE_AND_SLEEP_REJECTED = 0x19, // from-device, stateful suspend payload rejected + BCE_MB_SAVE_RESTORE_STATE_COMPLETE = 0x1A, // from-device +}; + +#define BCE_MB_MSG(type, value) (((u64) (type) << 58) | ((value) & 0x3FFFFFFFFFFFFFFLL)) +#define BCE_MB_TYPE(v) ((u32) (v >> 58)) +#define BCE_MB_VALUE(v) (v & 0x3FFFFFFFFFFFFFFLL) + +void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb); + +int bce_mailbox_channel_pause(struct bce_mailbox *mb); +void bce_mailbox_channel_resume(struct bce_mailbox *mb); + +int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv); +int bce_mailbox_send_timeout(struct bce_mailbox *mb, u64 msg, u64 *recv, unsigned int timeout_ms); +int bce_mailbox_send_no_reply(struct bce_mailbox *mb, u64 msg); +int bce_mailbox_send_locked(struct bce_mailbox *mb, u64 msg, u64 *recv); +int bce_mailbox_send_locked_timeout(struct bce_mailbox *mb, u64 msg, u64 *recv, unsigned int timeout_ms); +int bce_mailbox_send_no_reply_locked(struct bce_mailbox *mb, u64 msg); + +int bce_mailbox_handle_interrupt(struct bce_mailbox *mb); + + +struct bce_xhci_pm { + void __iomem *reg; + struct timer_list timer; + struct spinlock stop_sl; + bool stopped; +}; + +void bce_xhci_pm_init(struct bce_xhci_pm *pm, void __iomem *reg); + +void bce_xhci_pm_start(struct bce_xhci_pm *pm, bool is_initial); + +void bce_xhci_pm_stop(struct bce_xhci_pm *pm); + +#endif //BCEDRIVER_MAILBOX_H diff --git a/drivers/staging/t2bce/t2bce_core/t2bce.h b/drivers/staging/t2bce/t2bce_core/t2bce.h new file mode 100644 index 000000000..fe50f2ff7 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_core/t2bce.h @@ -0,0 +1,51 @@ +#ifndef APPLE_BCE_H +#define APPLE_BCE_H + +#include +#include +#include +#include +#include +#include "mailbox.h" +#include "t2bce_dma_queue.h" + +#define BC_PROTOCOL_VERSION 0x20001 + +struct t2bce_device { + struct pci_dev *pci, *pci0; + dev_t devt; + struct device *dev; + void __iomem *reg_mem_mb; + void __iomem *reg_mem_dma; + struct t2bce_dma_engine dma; + struct bce_mailbox mbox; + struct bce_xhci_pm xhci_pm; + bool is_being_removed; + + dma_addr_t saved_data_dma_addr; + void *saved_data_dma_ptr; + size_t saved_data_dma_size; + u32 fw_version; + bool stateful_suspend_valid; + bool no_state_fallback; + bool no_state_resume; + bool mailbox_channel_active; + struct mutex pm_lock; + struct mutex clients_lock; + struct list_head clients; + struct srcu_struct clients_srcu; + bool clients_srcu_ready; +}; + +extern struct t2bce_device *global_bce; + +void t2bce_core_notify_resume_complete(struct t2bce_device *bce); +void t2bce_core_clients_shutdown(struct t2bce_device *bce); +void t2bce_core_clients_pm_reset(struct t2bce_device *bce); +void t2bce_core_clients_pm_prepare(struct t2bce_device *bce); +void t2bce_core_clients_pm_prepare_no_state(struct t2bce_device *bce); +void t2bce_core_clients_pm_mark_no_state_resume(struct t2bce_device *bce); +bool t2bce_core_clients_pm_has_no_state_resume(struct t2bce_device *bce); +void t2bce_core_clients_pm_complete(struct t2bce_device *bce); + +#endif //APPLE_BCE_H diff --git a/drivers/staging/t2bce/t2bce_core/t2bce_main.c b/drivers/staging/t2bce/t2bce_core/t2bce_main.c new file mode 100644 index 000000000..6e90a7ea4 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_core/t2bce_main.c @@ -0,0 +1,766 @@ +#include "t2bce.h" +#include +#include +#include +#include + +/* + * Apple T2 Macs contain a coprocessor serving as a board supervisor. + * The T2 is basically a repurposed iPhone CPU running bridgeOS, + * a stripped down version of WatchOS. Apple reuses the audio codec + * and USB interfaces of this SOC and uses a "Buffer Copy Engine" (BCE) + * for communication between the host OS, the T2 and the VHCI USB devices + * behind the T2 USB hub like internal keyboard, trackpad, touchbar etc.. + * + * This core driver owns the BCE PCI device, mailbox, DMA queue registration + * and system PM ordering. Other modules bind as clients: + * - t2bce_audio exposes the bridgeOS audio endpoints to ALSA. + * - t2bce_vhci exposes the T2-internal virtual USB hub and devices to usbcore. + * - t2bce_dma provides the queue engine used by both. + * + * ACPI describes the host-side PCI devices, but bridgeOS still controls the + * remote endpoints and must be kept in sync during probe, shutdown and system + * suspend/resume. The PM code below mirrors the observed firmware ordering: + * It quiesces clients, drains the mailbox channel, asks bridgeOS to save or discard + * BCE state, then lets clients rebuild or continue according to the selected + * resume path. + */ + +static dev_t bce_chrdev; +static struct class *bce_class; +static const bool bce_use_stateful_sleep = true; + +struct t2bce_device *global_bce; + +static int bce_create_command_queues(struct t2bce_device *bce); +static void bce_free_command_queues(struct t2bce_device *bce); +static irqreturn_t bce_handle_mb_irq(int irq, void *dev); +static irqreturn_t bce_handle_dma_irq(int irq, void *dev); +static int bce_fw_version_handshake(struct t2bce_device *bce); +static int bce_register_command_queue(struct t2bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq); +static int t2bce_dma_register_queue(void *userdata, struct bce_queue_memcfg *cfg, + const char *name, bool isdirout); +static int t2bce_dma_unregister_queue(void *userdata, u16 qid); +static int t2bce_dma_flush_queue(void *userdata, u16 qid); +static int bce_alloc_state_buffer(struct t2bce_device *bce); +static void bce_free_state_buffer(struct t2bce_device *bce); +static int bce_pm_suspend_prepare(struct t2bce_device *bce); +static void bce_pm_suspend_abort(struct t2bce_device *bce); +static void bce_pm_resume_finish(struct t2bce_device *bce); + +static const struct t2bce_dma_engine_ops t2bce_dma_ops = { + .register_queue = t2bce_dma_register_queue, + .unregister_queue = t2bce_dma_unregister_queue, + .flush_queue = t2bce_dma_flush_queue, +}; + +static int bce_alloc_state_buffer(struct t2bce_device *bce) +{ + /* bridgeOS stateful sleep stores BCE state in a persistent 0x2000 buffer. */ + bce->saved_data_dma_size = 0x2000; + bce->saved_data_dma_ptr = dma_alloc_coherent(&bce->pci->dev, bce->saved_data_dma_size, + &bce->saved_data_dma_addr, GFP_KERNEL); + if (!bce->saved_data_dma_ptr) { + bce->saved_data_dma_size = 0; + bce->saved_data_dma_addr = 0; + return -ENOMEM; + } + + return 0; +} + +static void bce_free_state_buffer(struct t2bce_device *bce) +{ + if (!bce->saved_data_dma_ptr) + return; + + dma_free_coherent(&bce->pci->dev, bce->saved_data_dma_size, + bce->saved_data_dma_ptr, bce->saved_data_dma_addr); + bce->saved_data_dma_ptr = NULL; + bce->saved_data_dma_addr = 0; + bce->saved_data_dma_size = 0; +} + +static int t2bce_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + struct t2bce_device *bce = NULL; + int status = 0; + int nvec; + + pr_debug("t2bce_core: capturing our device\n"); + + if (pci_enable_device(dev)) + return -ENODEV; + if (pci_request_regions(dev, "t2bce_core")) { + status = -ENODEV; + goto fail; + } + pci_set_master(dev); + nvec = pci_alloc_irq_vectors(dev, 1, 8, PCI_IRQ_MSI); + if (nvec < 5) { + status = -EINVAL; + goto fail; + } + + bce = kzalloc(sizeof(struct t2bce_device), GFP_KERNEL); + if (!bce) { + status = -ENOMEM; + goto fail; + } + + bce->pci = dev; + pci_set_drvdata(dev, bce); + + bce->devt = bce_chrdev; + bce->dev = device_create(bce_class, &dev->dev, bce->devt, NULL, "t2bce_core"); + if (IS_ERR_OR_NULL(bce->dev)) { + status = PTR_ERR(bce->dev); + goto fail; + } + + bce->reg_mem_mb = pci_iomap(dev, 4, 0); + bce->reg_mem_dma = pci_iomap(dev, 2, 0); + + if (IS_ERR_OR_NULL(bce->reg_mem_mb) || IS_ERR_OR_NULL(bce->reg_mem_dma)) { + dev_warn(&dev->dev, "t2bce_core: Failed to pci_iomap required regions\n"); + goto fail; + } + + bce_mailbox_init(&bce->mbox, bce->reg_mem_mb); + bce_xhci_pm_init(&bce->xhci_pm, bce->reg_mem_mb); + bce->dma.dma_dev = &dev->dev; + bce->dma.ops = &t2bce_dma_ops; + bce->dma.ops_userdata = bce; + bce->dma.reg_mem_dma = bce->reg_mem_dma; + + spin_lock_init(&bce->dma.queues_lock); + ida_init(&bce->dma.queue_ida); + mutex_init(&bce->pm_lock); + mutex_init(&bce->clients_lock); + INIT_LIST_HEAD(&bce->clients); + status = init_srcu_struct(&bce->clients_srcu); + if (status) + goto fail; + bce->clients_srcu_ready = true; + bce->mailbox_channel_active = true; + + if ((status = pci_request_irq(dev, 0, bce_handle_mb_irq, NULL, dev, "bce_mbox"))) + goto fail; + if ((status = pci_request_irq(dev, 4, NULL, bce_handle_dma_irq, dev, "bce_dma"))) + goto fail_interrupt_0; + + if ((status = dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(37)))) { + dev_warn(&dev->dev, "dma: Setting mask failed\n"); + goto fail_interrupt; + } + + if ((status = bce_alloc_state_buffer(bce))) + goto fail_interrupt; + + /* + * DMA on the BCE function depends on function 0 also being bus master. + * Keep a reference so probe and resume can restore that firmware-visible + * host state. + */ + bce->pci0 = pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0)); +#ifndef WITHOUT_NVME_PATCH + if ((status = pci_enable_device_mem(bce->pci0))) { + dev_warn(&dev->dev, "t2bce_core: failed to enable function 0\n"); + goto fail_dev0; + } +#endif + pci_set_master(bce->pci0); + + /* bridgeOS expects the XHCI PM doorbell before the controller becomes active. */ + bce_xhci_pm_start(&bce->xhci_pm, true); + + if ((status = bce_fw_version_handshake(bce))) + goto fail_ts; + pr_debug("t2bce_core: handshake done\n"); + + if ((status = bce_create_command_queues(bce))) { + pr_err("t2bce_core: Creating command queues failed\n"); + goto fail_ts; + } + + global_bce = bce; + pr_info("t2bce_core: initialized\n"); + + return 0; + +fail_ts: + bce_free_state_buffer(bce); + bce_xhci_pm_stop(&bce->xhci_pm); +#ifndef WITHOUT_NVME_PATCH + pci_disable_device(bce->pci0); +fail_dev0: +#endif + pci_dev_put(bce->pci0); +fail_interrupt: + pci_free_irq(dev, 4, dev); +fail_interrupt_0: + pci_free_irq(dev, 0, dev); +fail: + if (bce && bce->dev) { + device_destroy(bce_class, bce->devt); + + if (!IS_ERR_OR_NULL(bce->reg_mem_mb)) + pci_iounmap(dev, bce->reg_mem_mb); + if (!IS_ERR_OR_NULL(bce->reg_mem_dma)) + pci_iounmap(dev, bce->reg_mem_dma); + + if (bce->clients_srcu_ready) + cleanup_srcu_struct(&bce->clients_srcu); + + kfree(bce); + } + + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + + if (!status) + status = -EINVAL; + return status; +} + +static int bce_create_command_queues(struct t2bce_device *bce) +{ + int status; + struct bce_queue_memcfg *cfg; + struct t2bce_dma_engine *dma = &bce->dma; + + dma->cmd_cq = t2bce_dma_alloc_cq(dma, 0, 0x20); + dma->cmd_cmdq = t2bce_dma_alloc_cmdq(dma, 1, 0x20); + if (dma->cmd_cq == NULL || dma->cmd_cmdq == NULL) { + status = -ENOMEM; + goto err; + } + dma->queues[0] = (struct bce_queue *) dma->cmd_cq; + dma->queues[1] = (struct bce_queue *) dma->cmd_cmdq->sq; + + cfg = kzalloc(sizeof(struct bce_queue_memcfg), GFP_KERNEL); + if (!cfg) { + status = -ENOMEM; + goto err; + } + t2bce_dma_get_cq_memcfg(dma->cmd_cq, cfg); + if ((status = bce_register_command_queue(bce, cfg, false))) + goto err; + t2bce_dma_get_sq_memcfg(dma->cmd_cmdq->sq, dma->cmd_cq, cfg); + if ((status = bce_register_command_queue(bce, cfg, true))) + goto err; + kfree(cfg); + + return 0; + +err: + if (dma->cmd_cq) + t2bce_dma_free_cq(dma, dma->cmd_cq); + if (dma->cmd_cmdq) + t2bce_dma_free_cmdq(dma, dma->cmd_cmdq); + return status; +} + +static void bce_free_command_queues(struct t2bce_device *bce) +{ + struct t2bce_dma_engine *dma = &bce->dma; + + t2bce_dma_free_cq(dma, dma->cmd_cq); + t2bce_dma_free_cmdq(dma, dma->cmd_cmdq); + dma->cmd_cq = NULL; + dma->queues[0] = NULL; +} + +static irqreturn_t bce_handle_mb_irq(int irq, void *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(dev); + bce_mailbox_handle_interrupt(&bce->mbox); + return IRQ_HANDLED; +} + +static irqreturn_t bce_handle_dma_irq(int irq, void *dev) +{ + int i; + size_t ce = 0; + struct t2bce_device *bce = pci_get_drvdata(dev); + struct t2bce_dma_engine *dma = &bce->dma; + + spin_lock(&dma->queues_lock); + for (i = 0; i < BCE_MAX_QUEUE_COUNT; i++) + if (dma->queues[i] && dma->queues[i]->type == BCE_QUEUE_CQ) + t2bce_dma_handle_cq_completions_locked(dma, (struct bce_queue_cq *) dma->queues[i], &ce); + spin_unlock(&dma->queues_lock); + t2bce_dma_dispatch_pending_sq_completions(dma, ce); + return IRQ_HANDLED; +} + +static int bce_fw_version_handshake(struct t2bce_device *bce) +{ + u64 result; + u32 fw_version; + int status; + + if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_SET_FW_PROTOCOL_VERSION, BC_PROTOCOL_VERSION), + &result))) + return status; + fw_version = (u32) BCE_MB_VALUE(result); + if (BCE_MB_TYPE(result) != BCE_MB_SET_FW_PROTOCOL_VERSION || + fw_version > BC_PROTOCOL_VERSION) { + pr_err("t2bce_core: FW version handshake failed %x:%llx\n", BCE_MB_TYPE(result), BCE_MB_VALUE(result)); + return -EINVAL; + } + bce->fw_version = fw_version; + return 0; +} + +static bool bce_stateful_supported(struct t2bce_device *bce) +{ + /* bridgeOS exposes stateful sleep through protocol version 0x20001+. */ + return bce_use_stateful_sleep && bce->fw_version >= BC_PROTOCOL_VERSION; +} + +static int bce_pm_wait_mailbox_idle(struct t2bce_device *bce) +{ + int retries = 100; + + while (atomic_read(&bce->mbox.mb_status) != 0) { + if (!--retries) { + pr_err("t2bce_core: timeout waiting for BCE mailbox channel to quiesce\n"); + return -ETIMEDOUT; + } + usleep_range(5000, 10000); + } + + return 0; +} + +static int bce_pm_channel_pause(struct t2bce_device *bce) +{ + int status; + + status = bce_mailbox_channel_pause(&bce->mbox); + if (status) + return status; + + status = bce_pm_wait_mailbox_idle(bce); + + if (status) + bce_mailbox_channel_resume(&bce->mbox); + if (status) + return status; + + /* Keep new mailbox commands out while bridgeOS is entering sleep. */ + bce->mailbox_channel_active = false; + return 0; +} + +static void bce_pm_channel_resume(struct t2bce_device *bce) +{ + /* Reopen mailbox traffic after bridgeOS completed the wake handshake. */ + bce->mailbox_channel_active = true; + bce_mailbox_channel_resume(&bce->mbox); +} + +static int bce_pm_suspend_prepare(struct t2bce_device *bce) +{ + int status; + + status = bce_pm_channel_pause(bce); + if (status) + return status; + + /* The XHCI PM doorbell is part of the bridgeOS sleep ordering. */ + bce_xhci_pm_stop(&bce->xhci_pm); + return 0; +} + +static void bce_pm_suspend_abort(struct t2bce_device *bce) +{ + /* Failed suspend must unwind the wrapper order back to the running state. */ + bce_xhci_pm_start(&bce->xhci_pm, false); + bce_pm_channel_resume(bce); +} + +static void bce_pm_resume_finish(struct t2bce_device *bce) +{ + /* Match the bridgeOS wake ordering before clients observe resume complete. */ + bce_xhci_pm_start(&bce->xhci_pm, false); + bce_pm_channel_resume(bce); +} + +static int bce_register_command_queue(struct t2bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq) +{ + int status; + int cmd_type; + u64 result; + /* The firmware accepts a TO_DEVICE mapping for this command payload. */ + dma_addr_t a = dma_map_single(&bce->pci->dev, cfg, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); + if (dma_mapping_error(&bce->pci->dev, a)) + return -ENOMEM; + cmd_type = is_sq ? BCE_MB_REGISTER_COMMAND_SQ : BCE_MB_REGISTER_COMMAND_CQ; + status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(cmd_type, a), &result); + dma_unmap_single(&bce->pci->dev, a, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE); + if (status) + return status; + if (BCE_MB_TYPE(result) != BCE_MB_REGISTER_COMMAND_QUEUE_REPLY) + return -EINVAL; + return 0; +} + +static int t2bce_dma_register_queue(void *userdata, struct bce_queue_memcfg *cfg, + const char *name, bool isdirout) +{ + struct t2bce_device *bce = userdata; + + return t2bce_dma_cmd_register_queue(bce->dma.cmd_cmdq, cfg, name, isdirout); +} + +static int t2bce_dma_unregister_queue(void *userdata, u16 qid) +{ + struct t2bce_device *bce = userdata; + + return t2bce_dma_cmd_unregister_memory_queue(bce->dma.cmd_cmdq, qid); +} + +static int t2bce_dma_flush_queue(void *userdata, u16 qid) +{ + struct t2bce_device *bce = userdata; + + return t2bce_dma_cmd_flush_memory_queue(bce->dma.cmd_cmdq, qid); +} + +static void t2bce_remove(struct pci_dev *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(dev); + bce->is_being_removed = true; + bce->dma.is_being_removed = true; + global_bce = NULL; + + bce_free_state_buffer(bce); + bce_xhci_pm_stop(&bce->xhci_pm); +#ifndef WITHOUT_NVME_PATCH + pci_disable_device(bce->pci0); +#endif + pci_dev_put(bce->pci0); + pci_free_irq(dev, 0, dev); + pci_free_irq(dev, 4, dev); + bce_free_command_queues(bce); + pci_iounmap(dev, bce->reg_mem_mb); + pci_iounmap(dev, bce->reg_mem_dma); + device_destroy(bce_class, bce->devt); + pci_free_irq_vectors(dev); + pci_release_regions(dev); + pci_disable_device(dev); + cleanup_srcu_struct(&bce->clients_srcu); + kfree(bce); +} + +static void t2bce_shutdown(struct pci_dev *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(dev); + int status = 0; + + if (!bce) + return; + + mutex_lock(&bce->pm_lock); + bce->is_being_removed = true; + bce->dma.is_being_removed = true; + bce->stateful_suspend_valid = false; + bce->no_state_fallback = false; + bce->no_state_resume = false; + t2bce_core_clients_pm_reset(bce); + + /* + * Do not tear down allocations here; + * just leave the T2 side quiet while command queues and mailbox access are + * still valid: USB HCD off, XHCI PM sentinel written, mailbox drained. + */ + t2bce_core_clients_shutdown(bce); + bce_xhci_pm_stop(&bce->xhci_pm); + + if (bce->mailbox_channel_active) { + status = bce_pm_channel_pause(bce); + if (status) + pr_warn("t2bce_core: shutdown mailbox quiesce failed: %d\n", status); + } + + mutex_unlock(&bce->pm_lock); +} + +static int bce_pm_suspend_fallback_no_state(struct t2bce_device *bce) +{ + pr_debug("t2bce_core: suspend: forcing SLEEP_NO_STATE (no reply expected)\n"); + if (bce_mailbox_send_no_reply_locked(&bce->mbox, BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0))) { + pr_err("t2bce_core: suspend: SLEEP_NO_STATE send failed\n"); + return -EIO; + } + return 0; +} + +static int bce_pm_suspend_try_state(struct t2bce_device *bce) +{ + int status; + u64 resp; + + /* Try the stateful bridgeOS path first; a reject falls back to no-state. */ + bce->stateful_suspend_valid = false; + + if (!bce->saved_data_dma_ptr || !bce->saved_data_dma_addr || !bce->saved_data_dma_size) { + pr_err("t2bce_core: suspend failed (persistent state buffer missing)\n"); + return -ENOMEM; + } + + BUG_ON((bce->saved_data_dma_addr % 4096) != 0); + status = bce_mailbox_send_locked(&bce->mbox, + BCE_MB_MSG(BCE_MB_SAVE_STATE_AND_SLEEP, + (bce->saved_data_dma_addr & ~(4096LLU - 1)) | (bce->saved_data_dma_size / 4096)), + &resp); + if (status) { + pr_err("t2bce_core: suspend failed (mailbox send)\n"); + return status; + } + + if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { + pr_debug("t2bce_core: suspend: remote response: restore state saved\n"); + bce->stateful_suspend_valid = true; + return 0; + } + + if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_STATE_AND_SLEEP_REJECTED) { + pr_err("t2bce_core: remote rejected stateful suspend payload\n"); + return -EAGAIN; + } + return -EINVAL; +} + +static int bce_pm_resume_no_state(struct t2bce_device *bce) +{ + int status; + u64 resp; + + if ((status = bce_mailbox_send_locked(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &resp))) { + pr_err("t2bce_core: resume with no state failed (mailbox send)\n"); + return status; + } + if (BCE_MB_TYPE(resp) != BCE_MB_RESTORE_NO_STATE) { + pr_err("t2bce_core: resume with no state failed (invalid device response)\n"); + return -EINVAL; + } + return 0; +} + +static int bce_pm_resume_stateful(struct t2bce_device *bce) +{ + int status; + u64 resp; + + if ((status = bce_mailbox_send_locked(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_STATE_AND_WAKE, + (bce->saved_data_dma_addr & ~(4096LLU - 1)) | (bce->saved_data_dma_size / 4096)), &resp))) { + pr_err("t2bce_core: resume with state failed (mailbox send)\n"); + return status; + } + if (BCE_MB_TYPE(resp) != BCE_MB_SAVE_RESTORE_STATE_COMPLETE) { + pr_err("t2bce_core: resume with state failed (invalid device response)\n"); + return -EINVAL; + } + + return 0; +} + +static int t2bce_suspend(struct device *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + int status; + + pr_debug("t2bce_core: suspend: entry\n"); + mutex_lock(&bce->pm_lock); + + bce->stateful_suspend_valid = false; + bce->no_state_fallback = false; + bce->no_state_resume = false; + t2bce_core_clients_pm_reset(bce); + t2bce_core_clients_pm_prepare(bce); + + status = bce_pm_suspend_prepare(bce); + if (status) + goto out_unlock; + + if (!bce_stateful_supported(bce)) { + /* No-state resume tears down and rebuilds clients that cannot preserve firmware state. */ + t2bce_core_clients_pm_prepare_no_state(bce); + status = bce_pm_suspend_fallback_no_state(bce); + if (!status) { + bce->no_state_fallback = true; + bce->no_state_resume = true; + t2bce_core_clients_pm_mark_no_state_resume(bce); + } else { + bce_pm_suspend_abort(bce); + } + goto out_unlock; + } + + status = bce_pm_suspend_try_state(bce); + if (!status) + goto out_unlock; + + if (status != -EAGAIN) { + bce_pm_suspend_abort(bce); + goto out_unlock; + } + + /* bridgeOS can reject state save; fall back to the no-state path then. */ + pr_debug("t2bce_core: suspend: stateful path not ready, falling back to no-state\n"); + t2bce_core_clients_pm_prepare_no_state(bce); + status = bce_pm_suspend_fallback_no_state(bce); + if (!status) { + bce->no_state_fallback = true; + bce->no_state_resume = true; + t2bce_core_clients_pm_mark_no_state_resume(bce); + } else { + bce_pm_suspend_abort(bce); + } + +out_unlock: + mutex_unlock(&bce->pm_lock); + pr_info("t2bce_core: suspend: exit status=%d stateful_valid=%d no_state_resume=%d no_state_fallback=%d\n", + status, bce->stateful_suspend_valid, bce->no_state_resume, bce->no_state_fallback); + return status; +} + +static int t2bce_resume(struct device *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + int status; + bool used_stateful; + + pr_debug("t2bce_core: resume: entry\n"); + mutex_lock(&bce->pm_lock); + + pci_set_master(bce->pci); + pci_set_master(bce->pci0); + + /* Resume follows the suspend result, not a preselected policy. */ + used_stateful = bce_stateful_supported(bce) && bce->stateful_suspend_valid; + pr_debug("t2bce_core: resume path: %s\n", used_stateful ? "stateful" : "no-state"); + if (used_stateful) + status = bce_pm_resume_stateful(bce); + else + status = bce_pm_resume_no_state(bce); + if (status) + goto out_unlock; + + if (used_stateful) + bce->stateful_suspend_valid = false; + + bce_pm_resume_finish(bce); + +out_unlock: + mutex_unlock(&bce->pm_lock); + pr_info("t2bce_core: resume: exit status=%d path=%s stateful_valid=%d no_state_resume=%d no_state_fallback=%d\n", + status, used_stateful ? "stateful" : "no-state", + bce->stateful_suspend_valid, bce->no_state_resume, bce->no_state_fallback); + return status; +} + +static int t2bce_prepare(struct device *dev) +{ + /* + * Force PCI PM to run our real suspend/resume callbacks instead of + * short-cutting an "already runtime suspended" device through complete(). + * The T2 mailbox save/restore handshake lives in those callbacks. + */ + return 0; +} + +static void t2bce_complete(struct device *dev) +{ + struct t2bce_device *bce = pci_get_drvdata(to_pci_dev(dev)); + + pr_debug("t2bce_core: complete: entry no_state_fallback=%d no_state_resume=%d\n", + bce->no_state_fallback, bce->no_state_resume); + if (bce->no_state_fallback && t2bce_core_clients_pm_has_no_state_resume(bce)) { + t2bce_core_clients_pm_complete(bce); + bce->no_state_fallback = false; + } + + t2bce_core_notify_resume_complete(bce); + bce->no_state_resume = false; + pr_debug("t2bce_core: complete: exit\n"); +} + +static struct pci_device_id t2bce_ids[ ] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1801) }, + { 0, }, +}; + +MODULE_DEVICE_TABLE(pci, t2bce_ids); + +struct dev_pm_ops t2bce_pci_driver_pm = { + .prepare = t2bce_prepare, + .suspend = t2bce_suspend, + .resume = t2bce_resume, + .complete = t2bce_complete +}; +struct pci_driver t2bce_pci_driver = { + .name = "t2bce_core", + .id_table = t2bce_ids, + .probe = t2bce_probe, + .remove = t2bce_remove, + .shutdown = t2bce_shutdown, + .driver = { + .pm = &t2bce_pci_driver_pm + } +}; + + +static int __init t2bce_module_init(void) +{ + int result; + + if ((result = alloc_chrdev_region(&bce_chrdev, 0, 1, "t2bce_core"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + bce_class = class_create(THIS_MODULE, "t2bce_core"); +#else + bce_class = class_create("t2bce_core"); +#endif + if (IS_ERR(bce_class)) { + result = PTR_ERR(bce_class); + goto fail_class; + } + result = pci_register_driver(&t2bce_pci_driver); + if (result) + goto fail_drv; + + pr_info("t2bce_core: module initialized\n"); + return 0; + +fail_drv: + pci_unregister_driver(&t2bce_pci_driver); +fail_class: + class_destroy(bce_class); +fail_chrdev: + unregister_chrdev_region(bce_chrdev, 1); + if (!result) + result = -EINVAL; + pr_info("t2bce_core: module init failed status=%d\n", result); + return result; +} +static void __exit t2bce_module_exit(void) +{ + pci_unregister_driver(&t2bce_pci_driver); + + class_destroy(bce_class); + unregister_chrdev_region(bce_chrdev, 1); + pr_info("t2bce_core: module exited\n"); +} + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("AndrĂ© Eikmeyer "); +MODULE_DESCRIPTION("T2 BCE core driver"); +MODULE_VERSION("0.06"); +MODULE_SOFTDEP("post: t2bce_vhci"); +module_init(t2bce_module_init); +module_exit(t2bce_module_exit); diff --git a/drivers/staging/t2bce/t2bce_core/transport.c b/drivers/staging/t2bce/t2bce_core/transport.c new file mode 100644 index 000000000..1f731faff --- /dev/null +++ b/drivers/staging/t2bce/t2bce_core/transport.c @@ -0,0 +1,408 @@ +#include "t2bce_core_transport.h" +#include "t2bce.h" + +#include +#include +#include +#include +#include + +struct t2bce_core_client { + struct t2bce_device *bce; + struct device *dev; + struct device_link *link; + struct list_head list; + t2bce_core_client_resume_callback resume_complete; + void *resume_complete_userdata; + struct t2bce_core_client_pm_ops pm_ops; + void *pm_userdata; +}; + +struct t2bce_sq_ctx { + t2bce_core_sq_completion completion; + void *userdata; +}; + +static struct bce_queue_cq *to_bce_cq(struct t2bce_core_queue_cq *cq) +{ + return (struct bce_queue_cq *) cq; +} + +static struct t2bce_core_queue_cq *to_t2bce_cq(struct bce_queue_cq *cq) +{ + return (struct t2bce_core_queue_cq *) cq; +} + +static struct bce_queue_sq *to_bce_sq(struct t2bce_core_queue_sq *sq) +{ + return (struct bce_queue_sq *) sq; +} + +static struct t2bce_core_queue_sq *to_t2bce_sq(struct bce_queue_sq *sq) +{ + return (struct t2bce_core_queue_sq *) sq; +} + +static void t2bce_sq_completion_adapter(struct bce_queue_sq *sq) +{ + struct t2bce_sq_ctx *ctx = sq->userdata; + + ctx->completion(to_t2bce_sq(sq)); +} + +struct t2bce_core_client *t2bce_core_client_get(struct device *dev) +{ + struct t2bce_device *bce = global_bce; + struct t2bce_core_client *client; + + if (!bce) + return ERR_PTR(-EPROBE_DEFER); + + if (bce->is_being_removed) + return ERR_PTR(-ENODEV); + + client = kzalloc(sizeof(*client), GFP_KERNEL); + if (!client) + return ERR_PTR(-ENOMEM); + + client->bce = bce; + client->dev = dev; + client->link = device_link_add(dev, &bce->pci->dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER); + if (!client->link) { + kfree(client); + return ERR_PTR(-ENODEV); + } + + mutex_lock(&bce->clients_lock); + list_add_tail_rcu(&client->list, &bce->clients); + mutex_unlock(&bce->clients_lock); + + return client; +} +EXPORT_SYMBOL_GPL(t2bce_core_client_get); + +void t2bce_core_client_put(struct t2bce_core_client *client) +{ + if (!client) + return; + + mutex_lock(&client->bce->clients_lock); + list_del_rcu(&client->list); + mutex_unlock(&client->bce->clients_lock); + + synchronize_srcu(&client->bce->clients_srcu); + + if (client->link) + device_link_del(client->link); + kfree(client); +} +EXPORT_SYMBOL_GPL(t2bce_core_client_put); + +struct device *t2bce_core_device_get(void) +{ + struct t2bce_device *bce = global_bce; + + if (!bce) + return ERR_PTR(-EPROBE_DEFER); + + if (bce->is_being_removed || !bce->dev) + return ERR_PTR(-ENODEV); + + return get_device(bce->dev); +} +EXPORT_SYMBOL_GPL(t2bce_core_device_get); + +void t2bce_core_device_put(struct device *dev) +{ + if (dev) + put_device(dev); +} +EXPORT_SYMBOL_GPL(t2bce_core_device_put); + +struct device *t2bce_core_client_dma_dev(struct t2bce_core_client *client) +{ + return &client->bce->pci->dev; +} +EXPORT_SYMBOL_GPL(t2bce_core_client_dma_dev); + +bool t2bce_core_client_no_state_resume(struct t2bce_core_client *client) +{ + return client->bce->no_state_resume; +} +EXPORT_SYMBOL_GPL(t2bce_core_client_no_state_resume); + +void t2bce_core_client_set_resume_complete_callback(struct t2bce_core_client *client, + t2bce_core_client_resume_callback callback, void *userdata) +{ + if (!callback) { + smp_store_release(&client->resume_complete, NULL); + WRITE_ONCE(client->resume_complete_userdata, NULL); + return; + } + + WRITE_ONCE(client->resume_complete_userdata, userdata); + smp_store_release(&client->resume_complete, callback); +} +EXPORT_SYMBOL_GPL(t2bce_core_client_set_resume_complete_callback); + +void t2bce_core_client_set_pm_ops(struct t2bce_core_client *client, + const struct t2bce_core_client_pm_ops *ops, void *userdata) +{ + if (!ops) { + memset(&client->pm_ops, 0, sizeof(client->pm_ops)); + WRITE_ONCE(client->pm_userdata, NULL); + return; + } + + WRITE_ONCE(client->pm_userdata, userdata); + client->pm_ops = *ops; +} +EXPORT_SYMBOL_GPL(t2bce_core_client_set_pm_ops); + +void t2bce_core_clients_shutdown(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.shutdown) + client->pm_ops.shutdown(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_clients_pm_reset(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_reset) + client->pm_ops.pm_reset(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_clients_pm_prepare(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_prepare) + client->pm_ops.pm_prepare(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_clients_pm_prepare_no_state(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_prepare_no_state) + client->pm_ops.pm_prepare_no_state(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_clients_pm_mark_no_state_resume(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_mark_no_state_resume) + client->pm_ops.pm_mark_no_state_resume(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +bool t2bce_core_clients_pm_has_no_state_resume(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + bool ret = false; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_is_no_state_resume && + client->pm_ops.pm_is_no_state_resume(READ_ONCE(client->pm_userdata))) { + ret = true; + break; + } + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); + return ret; +} + +void t2bce_core_clients_pm_complete(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + if (client->pm_ops.pm_complete) + client->pm_ops.pm_complete(READ_ONCE(client->pm_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +void t2bce_core_notify_resume_complete(struct t2bce_device *bce) +{ + struct t2bce_core_client *client; + int srcu_idx; + + srcu_idx = srcu_read_lock(&bce->clients_srcu); + list_for_each_entry_srcu(client, &bce->clients, list, + srcu_read_lock_held(&bce->clients_srcu)) { + t2bce_core_client_resume_callback callback = smp_load_acquire(&client->resume_complete); + + if (callback) + callback(READ_ONCE(client->resume_complete_userdata)); + } + srcu_read_unlock(&bce->clients_srcu, srcu_idx); +} + +struct t2bce_core_queue_cq *t2bce_core_create_cq(struct t2bce_core_client *client, u32 el_count) +{ + return to_t2bce_cq(t2bce_dma_create_cq(&client->bce->dma, el_count)); +} +EXPORT_SYMBOL_GPL(t2bce_core_create_cq); + +void t2bce_core_destroy_cq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq) +{ + t2bce_dma_destroy_cq(&client->bce->dma, to_bce_cq(cq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_destroy_cq); + +struct t2bce_core_queue_sq *t2bce_core_create_sq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq, + const char *name, u32 el_count, enum dma_data_direction direction, + t2bce_core_sq_completion compl, void *userdata) +{ + struct t2bce_sq_ctx *ctx; + struct bce_queue_sq *sq; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + + ctx->completion = compl; + ctx->userdata = userdata; + + sq = t2bce_dma_create_sq(&client->bce->dma, to_bce_cq(cq), name, el_count, direction, + t2bce_sq_completion_adapter, ctx); + if (!sq) { + kfree(ctx); + return NULL; + } + + return to_t2bce_sq(sq); +} +EXPORT_SYMBOL_GPL(t2bce_core_create_sq); + +void t2bce_core_destroy_sq(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq) +{ + struct bce_queue_sq *bce_sq = to_bce_sq(sq); + struct t2bce_sq_ctx *ctx = bce_sq->userdata; + + t2bce_dma_destroy_sq(&client->bce->dma, bce_sq); + kfree(ctx); +} +EXPORT_SYMBOL_GPL(t2bce_core_destroy_sq); + +void *t2bce_core_queue_sq_userdata(struct t2bce_core_queue_sq *sq) +{ + struct t2bce_sq_ctx *ctx = to_bce_sq(sq)->userdata; + + return ctx->userdata; +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_userdata); + +int t2bce_core_reserve_submission(struct t2bce_core_queue_sq *sq, unsigned long *timeout) +{ + return t2bce_dma_reserve_submission(to_bce_sq(sq), timeout); +} +EXPORT_SYMBOL_GPL(t2bce_core_reserve_submission); + +void t2bce_core_cancel_submission_reservation(struct t2bce_core_queue_sq *sq) +{ + t2bce_dma_cancel_submission_reservation(to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_cancel_submission_reservation); + +void t2bce_core_set_next_submission_single(struct t2bce_core_queue_sq *sq, dma_addr_t addr, size_t size) +{ + t2bce_dma_set_next_submission_single(to_bce_sq(sq), addr, size); +} +EXPORT_SYMBOL_GPL(t2bce_core_set_next_submission_single); + +void t2bce_core_set_next_submission_segment_list(struct t2bce_core_queue_sq *sq, + dma_addr_t segl_addr, size_t segl_size) +{ + t2bce_dma_set_next_submission_segment_list(to_bce_sq(sq), segl_addr, segl_size); +} +EXPORT_SYMBOL_GPL(t2bce_core_set_next_submission_segment_list); + +void t2bce_core_submit_to_device(struct t2bce_core_queue_sq *sq) +{ + t2bce_dma_submit_to_device(to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_submit_to_device); + +void t2bce_core_notify_submission_complete(struct t2bce_core_queue_sq *sq) +{ + t2bce_dma_notify_submission_complete(to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_notify_submission_complete); + +struct t2bce_core_sq_completion_data *t2bce_core_next_completion(struct t2bce_core_queue_sq *sq) +{ + return (struct t2bce_core_sq_completion_data *) t2bce_dma_next_completion(to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_next_completion); + +u32 t2bce_core_queue_sq_head(struct t2bce_core_queue_sq *sq) +{ + return to_bce_sq(sq)->head; +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_head); + +u32 t2bce_core_queue_sq_tail(struct t2bce_core_queue_sq *sq) +{ + return to_bce_sq(sq)->tail; +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_tail); + +u32 t2bce_core_queue_sq_available(struct t2bce_core_queue_sq *sq) +{ + return atomic_read(&to_bce_sq(sq)->available_commands); +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_available); + +u32 t2bce_core_queue_sq_capacity(struct t2bce_core_queue_sq *sq) +{ + return to_bce_sq(sq)->el_count; +} +EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_capacity); + +int t2bce_core_flush_queue(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq) +{ + return t2bce_dma_flush_sq(&client->bce->dma, to_bce_sq(sq)); +} +EXPORT_SYMBOL_GPL(t2bce_core_flush_queue); diff --git a/drivers/staging/t2bce/t2bce_dma/Makefile b/drivers/staging/t2bce/t2bce_dma/Makefile new file mode 100644 index 000000000..d8bb26e69 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_dma/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_DMA) += t2bce_dma.o +t2bce_dma-y := queue.o + +ccflags-y += -DWITHOUT_NVME_PATCH +ccflags-y += -I$(src)/include diff --git a/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h b/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h new file mode 100644 index 000000000..079582e89 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h @@ -0,0 +1,182 @@ +#ifndef T2BCE_DMA_QUEUE_H +#define T2BCE_DMA_QUEUE_H + +#include +#include + +#define BCE_CMD_SIZE 0x40 +#define BCE_MAX_QUEUE_COUNT 0x100 + +#define BCE_QUEUE_USER_MIN 2 +#define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1) + +struct t2bce_dma_segment { + u64 addr; + u64 length; +}; + +struct bce_queue_memcfg; + +struct t2bce_dma_engine_ops { + int (*register_queue)(void *userdata, struct bce_queue_memcfg *cfg, + const char *name, bool isdirout); + int (*unregister_queue)(void *userdata, u16 qid); + int (*flush_queue)(void *userdata, u16 qid); +}; + +enum bce_queue_type { + BCE_QUEUE_CQ, BCE_QUEUE_SQ +}; +struct bce_queue { + int qid; + int type; +}; +struct bce_queue_cq { + int qid; + int type; + u32 el_count; + dma_addr_t dma_handle; + void *data; + + u32 index; +}; +struct bce_queue_sq; +typedef void (*bce_sq_completion)(struct bce_queue_sq *q); +struct bce_sq_completion_data { + u32 status; + u64 data_size; + u64 result; +}; +struct bce_queue_sq { + int qid; + int type; + u32 el_size; + u32 el_count; + dma_addr_t dma_handle; + void *data; + void *userdata; + void __iomem *reg_mem_dma; + + atomic_t available_commands; + struct completion available_command_completion; + atomic_t available_command_completion_waiting_count; + u32 head, tail; + + u32 completion_cidx, completion_tail; + struct bce_sq_completion_data *completion_data; + bool has_pending_completions; + bce_sq_completion completion; +}; + +struct bce_queue_cmdq_result_el { + struct completion cmpl; + u32 status; + u64 result; + u32 slot; + u32 generation; +}; +struct bce_queue_cmdq { + struct bce_queue_sq *sq; + struct spinlock lck; + struct bce_queue_cmdq_result_el **tres; + u32 *slot_gen; +}; + +struct t2bce_dma_engine { + struct device *dma_dev; + const struct t2bce_dma_engine_ops *ops; + void *ops_userdata; + void __iomem *reg_mem_dma; + struct bce_queue *queues[BCE_MAX_QUEUE_COUNT]; + struct spinlock queues_lock; + struct ida queue_ida; + struct bce_queue_cq *cmd_cq; + struct bce_queue_cmdq *cmd_cmdq; + struct bce_queue_sq *int_sq_list[BCE_MAX_QUEUE_COUNT]; + bool is_being_removed; +}; + +struct bce_queue_memcfg { + u16 qid; + u16 el_count; + u16 vector_or_cq; + u16 _pad; + u64 addr; + u64 length; +}; + +enum bce_qe_completion_status { + BCE_COMPLETION_SUCCESS = 0, + BCE_COMPLETION_ERROR = 1, + BCE_COMPLETION_ABORTED = 2, + BCE_COMPLETION_NO_SPACE = 3, + BCE_COMPLETION_OVERRUN = 4 +}; +enum bce_qe_completion_flags { + BCE_COMPLETION_FLAG_PENDING = 0x8000 +}; +struct bce_qe_completion { + u64 result; + u64 data_size; + u16 qid; + u16 completion_index; + u16 status; // bce_qe_completion_status + u16 flags; // bce_qe_completion_flags +}; + +static __always_inline void *t2bce_dma_sq_element(struct bce_queue_sq *q, int i) { + return (void *) ((u8 *) q->data + q->el_size * i); +} +static __always_inline void *t2bce_dma_cq_element(struct bce_queue_cq *q, int i) { + return (void *) ((struct bce_qe_completion *) q->data + i); +} + +static __always_inline struct bce_sq_completion_data *t2bce_dma_next_completion(struct bce_queue_sq *sq) { + struct bce_sq_completion_data *res; + rmb(); + if (sq->completion_cidx == sq->completion_tail) + return NULL; + res = &sq->completion_data[sq->completion_cidx]; + sq->completion_cidx = (sq->completion_cidx + 1) % sq->el_count; + return res; +} + +struct bce_queue_cq *t2bce_dma_alloc_cq(struct t2bce_dma_engine *dma, int qid, u32 el_count); +void t2bce_dma_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); +void t2bce_dma_free_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq); +void t2bce_dma_handle_cq_completions_locked(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, size_t *ce); +void t2bce_dma_dispatch_pending_sq_completions(struct t2bce_dma_engine *dma, size_t ce); + +struct bce_queue_sq *t2bce_dma_alloc_sq(struct t2bce_dma_engine *dma, int qid, u32 el_size, u32 el_count, + bce_sq_completion compl, void *userdata); +void t2bce_dma_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg); +void t2bce_dma_free_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq); +int t2bce_dma_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout); +void t2bce_dma_cancel_submission_reservation(struct bce_queue_sq *sq); +void t2bce_dma_submit_to_device(struct bce_queue_sq *sq); +void t2bce_dma_notify_submission_complete(struct bce_queue_sq *sq); + +void t2bce_dma_set_next_submission_single(struct bce_queue_sq *sq, dma_addr_t addr, size_t size); +void t2bce_dma_set_next_submission_segment_list(struct bce_queue_sq *sq, + dma_addr_t segl_addr, size_t segl_size); + +struct bce_queue_cmdq *t2bce_dma_alloc_cmdq(struct t2bce_dma_engine *dma, int qid, u32 el_count); +void t2bce_dma_free_cmdq(struct t2bce_dma_engine *dma, struct bce_queue_cmdq *cmdq); + +u32 t2bce_dma_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout); +u32 t2bce_dma_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); +u32 t2bce_dma_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid); + + +/* User API - Creates and registers the queue */ + +struct bce_queue_cq *t2bce_dma_create_cq(struct t2bce_dma_engine *dma, u32 el_count); +struct bce_queue_sq *t2bce_dma_create_sq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, u32 el_count, + int direction, bce_sq_completion compl, void *userdata); +struct bce_queue_sq *t2bce_dma_create_sq_with_flags(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, + u32 el_count, u16 flags, bce_sq_completion compl, void *userdata); +void t2bce_dma_destroy_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq); +void t2bce_dma_destroy_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq); +int t2bce_dma_flush_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq); + +#endif diff --git a/drivers/staging/t2bce/t2bce_dma/queue.c b/drivers/staging/t2bce/t2bce_dma/queue.c new file mode 100644 index 000000000..0328e40cd --- /dev/null +++ b/drivers/staging/t2bce/t2bce_dma/queue.c @@ -0,0 +1,625 @@ +#include "t2bce_dma_queue.h" +#include +#include +#include +#include +#include +#include + +#define REG_DOORBELL_BASE 0x44000 + +struct bce_qe_submission { + u64 length; + u64 addr; + + u64 segl_addr; + u64 segl_length; +}; + +enum bce_submission_type { + BCE_SUBMISSION_SINGLE, + BCE_SUBMISSION_SEGMENT_LIST, +}; + +struct bce_submission { + enum bce_submission_type type; + union { + struct { + dma_addr_t addr; + size_t size; + } single; + struct { + dma_addr_t addr; + size_t size; + } segment_list; + }; +}; + +enum bce_cmdq_command { + BCE_CMD_REGISTER_MEMORY_QUEUE = 0x20, + BCE_CMD_UNREGISTER_MEMORY_QUEUE = 0x30, + BCE_CMD_FLUSH_MEMORY_QUEUE = 0x40, + BCE_CMD_SET_MEMORY_QUEUE_PROPERTY = 0x50 +}; +struct bce_cmdq_simple_memory_queue_cmd { + u16 cmd; // bce_cmdq_command + u16 flags; + u16 qid; +}; +struct bce_cmdq_register_memory_queue_cmd { + u16 cmd; // bce_cmdq_command + u16 flags; + u16 qid; + u16 _pad; + u16 el_count; + u16 vector_or_cq; + u16 _pad2; + u16 name_len; + char name[0x20]; + u64 addr; + u64 length; +}; + +struct bce_queue_cq *t2bce_dma_alloc_cq(struct t2bce_dma_engine *dma, int qid, u32 el_count) +{ + struct bce_queue_cq *q; + q = kzalloc(sizeof(struct bce_queue_cq), GFP_KERNEL); + if (!q) + return NULL; + q->qid = qid; + q->type = BCE_QUEUE_CQ; + q->el_count = el_count; + q->data = dma_alloc_coherent(dma->dma_dev, el_count * sizeof(struct bce_qe_completion), + &q->dma_handle, GFP_KERNEL); + if (!q->data) { + pr_err("DMA queue memory alloc failed\n"); + kfree(q); + return NULL; + } + return q; +} +EXPORT_SYMBOL_GPL(t2bce_dma_alloc_cq); + +void t2bce_dma_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) +{ + cfg->qid = (u16) cq->qid; + cfg->el_count = (u16) cq->el_count; + cfg->vector_or_cq = 0; + cfg->_pad = 0; + cfg->addr = cq->dma_handle; + cfg->length = cq->el_count * sizeof(struct bce_qe_completion); +} +EXPORT_SYMBOL_GPL(t2bce_dma_get_cq_memcfg); + +void t2bce_dma_free_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq) +{ + dma_free_coherent(dma->dma_dev, cq->el_count * sizeof(struct bce_qe_completion), cq->data, cq->dma_handle); + kfree(cq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_free_cq); + +static void t2bce_dma_handle_cq_completion(struct t2bce_dma_engine *dma, struct bce_qe_completion *e, size_t *ce) +{ + struct bce_queue *target; + struct bce_queue_sq *target_sq; + struct bce_sq_completion_data *cmpl; + if (e->qid >= BCE_MAX_QUEUE_COUNT) { + pr_err("Device sent a response for qid (%u) >= BCE_MAX_QUEUE_COUNT\n", e->qid); + return; + } + target = dma->queues[e->qid]; + if (!target || target->type != BCE_QUEUE_SQ) { + pr_err("Device sent a response for qid (%u), which does not exist\n", e->qid); + return; + } + target_sq = (struct bce_queue_sq *) target; + if (target_sq->completion_tail != e->completion_index) { + pr_err("Completion index mismatch; this is likely going to make this driver unusable\n"); + return; + } + if (!target_sq->has_pending_completions) { + target_sq->has_pending_completions = true; + dma->int_sq_list[(*ce)++] = target_sq; + } + cmpl = &target_sq->completion_data[e->completion_index]; + cmpl->status = e->status; + cmpl->data_size = e->data_size; + cmpl->result = e->result; + wmb(); + target_sq->completion_tail = (target_sq->completion_tail + 1) % target_sq->el_count; +} + +void t2bce_dma_handle_cq_completions_locked(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, size_t *ce) +{ + struct bce_qe_completion *e; + e = t2bce_dma_cq_element(cq, cq->index); + if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) + return; + mb(); + while (true) { + e = t2bce_dma_cq_element(cq, cq->index); + if (!(e->flags & BCE_COMPLETION_FLAG_PENDING)) + break; + pr_debug("t2bce_dma: compl: %i: %i %llx %llx", e->qid, e->status, e->data_size, e->result); + t2bce_dma_handle_cq_completion(dma, e, ce); + e->flags = 0; + cq->index = (cq->index + 1) % cq->el_count; + } + mb(); + iowrite32(cq->index, (u32 *) ((u8 *) dma->reg_mem_dma + REG_DOORBELL_BASE) + cq->qid); +} +EXPORT_SYMBOL_GPL(t2bce_dma_handle_cq_completions_locked); + +void t2bce_dma_dispatch_pending_sq_completions(struct t2bce_dma_engine *dma, size_t ce) +{ + struct bce_queue_sq *sq; + + while (ce) { + --ce; + sq = dma->int_sq_list[ce]; + sq->completion(sq); + sq->has_pending_completions = false; + } +} +EXPORT_SYMBOL_GPL(t2bce_dma_dispatch_pending_sq_completions); + + +struct bce_queue_sq *t2bce_dma_alloc_sq(struct t2bce_dma_engine *dma, int qid, u32 el_size, u32 el_count, + bce_sq_completion compl, void *userdata) +{ + struct bce_queue_sq *q; + q = kzalloc(sizeof(struct bce_queue_sq), GFP_KERNEL); + if (!q) + return NULL; + q->qid = qid; + q->type = BCE_QUEUE_SQ; + q->el_size = el_size; + q->el_count = el_count; + q->data = dma_alloc_coherent(dma->dma_dev, el_count * el_size, + &q->dma_handle, GFP_KERNEL); + q->completion = compl; + q->userdata = userdata; + q->completion_data = kzalloc(sizeof(struct bce_sq_completion_data) * el_count, GFP_KERNEL); + q->reg_mem_dma = dma->reg_mem_dma; + atomic_set(&q->available_commands, el_count - 1); + init_completion(&q->available_command_completion); + atomic_set(&q->available_command_completion_waiting_count, 0); + if (!q->data || !q->completion_data) { + pr_err("DMA queue memory alloc failed\n"); + if (q->data) + dma_free_coherent(dma->dma_dev, el_count * el_size, q->data, q->dma_handle); + kfree(q->completion_data); + kfree(q); + return NULL; + } + return q; +} +EXPORT_SYMBOL_GPL(t2bce_dma_alloc_sq); + +void t2bce_dma_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg) +{ + cfg->qid = (u16) sq->qid; + cfg->el_count = (u16) sq->el_count; + cfg->vector_or_cq = (u16) cq->qid; + cfg->_pad = 0; + cfg->addr = sq->dma_handle; + cfg->length = sq->el_count * sq->el_size; +} +EXPORT_SYMBOL_GPL(t2bce_dma_get_sq_memcfg); + +void t2bce_dma_free_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq) +{ + dma_free_coherent(dma->dma_dev, sq->el_count * sq->el_size, sq->data, sq->dma_handle); + kfree(sq->completion_data); + kfree(sq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_free_sq); + +int t2bce_dma_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout) +{ + while (atomic_dec_if_positive(&sq->available_commands) < 0) { + if (!timeout || !*timeout) + return -EAGAIN; + atomic_inc(&sq->available_command_completion_waiting_count); + *timeout = wait_for_completion_timeout(&sq->available_command_completion, *timeout); + if (!*timeout) { + if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) < 0) + try_wait_for_completion(&sq->available_command_completion); /* consume the pending completion */ + } + } + return 0; +} +EXPORT_SYMBOL_GPL(t2bce_dma_reserve_submission); + +void t2bce_dma_cancel_submission_reservation(struct bce_queue_sq *sq) +{ + atomic_inc(&sq->available_commands); +} +EXPORT_SYMBOL_GPL(t2bce_dma_cancel_submission_reservation); + +static void *t2bce_dma_next_submission(struct bce_queue_sq *sq) +{ + void *ret = t2bce_dma_sq_element(sq, sq->tail); + sq->tail = (sq->tail + 1) % sq->el_count; + return ret; +} + +void t2bce_dma_submit_to_device(struct bce_queue_sq *sq) +{ + mb(); + iowrite32(sq->tail, (u32 *) ((u8 *) sq->reg_mem_dma + REG_DOORBELL_BASE) + sq->qid); +} +EXPORT_SYMBOL_GPL(t2bce_dma_submit_to_device); + +void t2bce_dma_notify_submission_complete(struct bce_queue_sq *sq) +{ + sq->head = (sq->head + 1) % sq->el_count; + atomic_inc(&sq->available_commands); + if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) >= 0) { + complete(&sq->available_command_completion); + } +} +EXPORT_SYMBOL_GPL(t2bce_dma_notify_submission_complete); + +static void t2bce_dma_write_submission(struct bce_qe_submission *element, const struct bce_submission *submission) +{ + switch (submission->type) { + case BCE_SUBMISSION_SINGLE: + element->addr = submission->single.addr; + element->length = submission->single.size; + element->segl_addr = 0; + element->segl_length = 0; + break; + case BCE_SUBMISSION_SEGMENT_LIST: + element->addr = 0; + element->length = 0; + element->segl_addr = submission->segment_list.addr; + element->segl_length = submission->segment_list.size; + break; + } +} + +void t2bce_dma_set_next_submission_single(struct bce_queue_sq *sq, dma_addr_t addr, size_t size) +{ + struct bce_submission submission = { + .type = BCE_SUBMISSION_SINGLE, + .single = { + .addr = addr, + .size = size, + }, + }; + + t2bce_dma_write_submission(t2bce_dma_next_submission(sq), &submission); +} +EXPORT_SYMBOL_GPL(t2bce_dma_set_next_submission_single); + +void t2bce_dma_set_next_submission_segment_list(struct bce_queue_sq *sq, + dma_addr_t segl_addr, size_t segl_size) +{ + struct bce_submission submission = { + .type = BCE_SUBMISSION_SEGMENT_LIST, + .segment_list = { + .addr = segl_addr, + .size = segl_size, + }, + }; + + t2bce_dma_write_submission(t2bce_dma_next_submission(sq), &submission); +} +EXPORT_SYMBOL_GPL(t2bce_dma_set_next_submission_segment_list); + +static void t2bce_dma_cmdq_completion(struct bce_queue_sq *q); + +struct bce_queue_cmdq *t2bce_dma_alloc_cmdq(struct t2bce_dma_engine *dma, int qid, u32 el_count) +{ + struct bce_queue_cmdq *q; + q = kzalloc(sizeof(struct bce_queue_cmdq), GFP_KERNEL); + if (!q) + return NULL; + q->sq = t2bce_dma_alloc_sq(dma, qid, BCE_CMD_SIZE, el_count, t2bce_dma_cmdq_completion, q); + if (!q->sq) { + kfree(q); + return NULL; + } + spin_lock_init(&q->lck); + q->tres = kzalloc(sizeof(struct bce_queue_cmdq_result_el*) * el_count, GFP_KERNEL); + if (!q->tres) { + t2bce_dma_free_sq(dma, q->sq); + kfree(q); + return NULL; + } + q->slot_gen = kzalloc(sizeof(u32) * el_count, GFP_KERNEL); + if (!q->slot_gen) { + kfree(q->tres); + t2bce_dma_free_sq(dma, q->sq); + kfree(q); + return NULL; + } + return q; +} +EXPORT_SYMBOL_GPL(t2bce_dma_alloc_cmdq); + +void t2bce_dma_free_cmdq(struct t2bce_dma_engine *dma, struct bce_queue_cmdq *cmdq) +{ + t2bce_dma_free_sq(dma, cmdq->sq); + kfree(cmdq->slot_gen); + kfree(cmdq->tres); + kfree(cmdq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_free_cmdq); + +void t2bce_dma_cmdq_completion(struct bce_queue_sq *q) +{ + struct bce_queue_cmdq_result_el *el; + struct bce_queue_cmdq *cmdq = q->userdata; + struct bce_sq_completion_data *result; + + spin_lock(&cmdq->lck); + while ((result = t2bce_dma_next_completion(q))) { + el = cmdq->tres[cmdq->sq->head]; + if (el && el->generation == cmdq->slot_gen[cmdq->sq->head]) { + el->result = result->result; + el->status = result->status; + mb(); + complete(&el->cmpl); + } + cmdq->tres[cmdq->sq->head] = NULL; + t2bce_dma_notify_submission_complete(q); + } + spin_unlock(&cmdq->lck); +} + +static __always_inline void *t2bce_dma_cmd_start(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) +{ + void *ret; + unsigned long timeout; + init_completion(&res->cmpl); + mb(); + + timeout = msecs_to_jiffies(1000L * 60 * 5); /* wait for up to ~5 minutes */ + if (t2bce_dma_reserve_submission(cmdq->sq, &timeout)) + return NULL; + + spin_lock(&cmdq->lck); + res->slot = cmdq->sq->tail; + res->generation = cmdq->slot_gen[cmdq->sq->tail]; + cmdq->tres[cmdq->sq->tail] = res; + ret = t2bce_dma_next_submission(cmdq->sq); + return ret; +} + +static __always_inline int t2bce_dma_cmd_finish(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res) +{ + t2bce_dma_submit_to_device(cmdq->sq); + spin_unlock(&cmdq->lck); + + if (!wait_for_completion_timeout(&res->cmpl, msecs_to_jiffies(5000))) { + pr_err("t2bce_dma: command queue timeout (slot %u)\n", res->slot); + spin_lock(&cmdq->lck); + cmdq->tres[res->slot] = NULL; + cmdq->slot_gen[res->slot]++; + spin_unlock(&cmdq->lck); + t2bce_dma_notify_submission_complete(cmdq->sq); + return -ETIMEDOUT; + } + mb(); + return 0; +} + +u32 t2bce_dma_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_register_memory_queue_cmd *cmd = t2bce_dma_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_REGISTER_MEMORY_QUEUE; + cmd->flags = (u16) ((name ? 2 : 0) | (isdirout ? 1 : 0)); + cmd->qid = cfg->qid; + cmd->el_count = cfg->el_count; + cmd->vector_or_cq = cfg->vector_or_cq; + memset(cmd->name, 0, sizeof(cmd->name)); + if (name) { + cmd->name_len = (u16) min(strlen(name), (size_t) sizeof(cmd->name)); + memcpy(cmd->name, name, cmd->name_len); + } else { + cmd->name_len = 0; + } + cmd->addr = cfg->addr; + cmd->length = cfg->length; + + if (t2bce_dma_cmd_finish(cmdq, &res)) + return (u32)-1; + return res.status; +} +EXPORT_SYMBOL_GPL(t2bce_dma_cmd_register_queue); + +u32 t2bce_dma_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_simple_memory_queue_cmd *cmd = t2bce_dma_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_UNREGISTER_MEMORY_QUEUE; + cmd->flags = 0; + cmd->qid = qid; + if (t2bce_dma_cmd_finish(cmdq, &res)) + return (u32)-1; + return res.status; +} +EXPORT_SYMBOL_GPL(t2bce_dma_cmd_unregister_memory_queue); + +u32 t2bce_dma_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid) +{ + struct bce_queue_cmdq_result_el res; + struct bce_cmdq_simple_memory_queue_cmd *cmd = t2bce_dma_cmd_start(cmdq, &res); + if (!cmd) + return (u32) -1; + cmd->cmd = BCE_CMD_FLUSH_MEMORY_QUEUE; + cmd->flags = 0; + cmd->qid = qid; + if (t2bce_dma_cmd_finish(cmdq, &res)) + return (u32)-1; + return res.status; +} +EXPORT_SYMBOL_GPL(t2bce_dma_cmd_flush_memory_queue); + +static int t2bce_dma_register_queue(struct t2bce_dma_engine *dma, struct bce_queue_memcfg *cfg, + const char *name, bool isdirout) +{ + if (!dma->ops || !dma->ops->register_queue) + return -ENODEV; + + return dma->ops->register_queue(dma->ops_userdata, cfg, name, isdirout); +} + +static int t2bce_dma_unregister_queue(struct t2bce_dma_engine *dma, u16 qid) +{ + if (!dma->ops || !dma->ops->unregister_queue) + return -ENODEV; + + return dma->ops->unregister_queue(dma->ops_userdata, qid); +} + +static int t2bce_dma_flush_queue(struct t2bce_dma_engine *dma, u16 qid) +{ + if (!dma->ops || !dma->ops->flush_queue) + return -ENODEV; + + return dma->ops->flush_queue(dma->ops_userdata, qid); +} + + +struct bce_queue_cq *t2bce_dma_create_cq(struct t2bce_dma_engine *dma, u32 el_count) +{ + struct bce_queue_cq *cq; + struct bce_queue_memcfg cfg; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + int qid = ida_simple_get(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); +#else + int qid = ida_alloc_range(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX - 1, GFP_KERNEL); +#endif + if (qid < 0) + return NULL; + cq = t2bce_dma_alloc_cq(dma, qid, el_count); + if (!cq) + return NULL; + t2bce_dma_get_cq_memcfg(cq, &cfg); + if (t2bce_dma_register_queue(dma, &cfg, NULL, false) != 0) { + pr_err("t2bce_dma: CQ registration failed (%i)", qid); + t2bce_dma_free_cq(dma, cq); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + ida_simple_remove(&dma->queue_ida, (uint) qid); +#else + ida_free(&dma->queue_ida, (uint) qid); +#endif + return NULL; + } + dma->queues[qid] = (struct bce_queue *) cq; + return cq; +} +EXPORT_SYMBOL_GPL(t2bce_dma_create_cq); + +struct bce_queue_sq *t2bce_dma_create_sq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, u32 el_count, + int direction, bce_sq_completion compl, void *userdata) +{ + struct bce_queue_sq *sq; + struct bce_queue_memcfg cfg; + int qid; + if (cq == NULL) + return NULL; /* cq can not be null */ + if (name == NULL) + return NULL; /* name can not be null */ + if (direction != DMA_TO_DEVICE && direction != DMA_FROM_DEVICE) + return NULL; /* unsupported direction */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + qid = ida_simple_get(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL); +#else + qid = ida_alloc_range(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX - 1, GFP_KERNEL); +#endif + if (qid < 0) + return NULL; + sq = t2bce_dma_alloc_sq(dma, qid, sizeof(struct bce_qe_submission), el_count, compl, userdata); + if (!sq) + return NULL; + t2bce_dma_get_sq_memcfg(sq, cq, &cfg); + if (t2bce_dma_register_queue(dma, &cfg, name, direction != DMA_FROM_DEVICE) != 0) { + pr_err("t2bce_dma: SQ registration failed (%i)", qid); + t2bce_dma_free_sq(dma, sq); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + ida_simple_remove(&dma->queue_ida, (uint) qid); +#else + ida_free(&dma->queue_ida, (uint) qid); +#endif + return NULL; + } + spin_lock(&dma->queues_lock); + dma->queues[qid] = (struct bce_queue *) sq; + spin_unlock(&dma->queues_lock); + return sq; +} +EXPORT_SYMBOL_GPL(t2bce_dma_create_sq); + +struct bce_queue_sq *t2bce_dma_create_sq_with_flags(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, + u32 el_count, u16 flags, bce_sq_completion compl, void *userdata) +{ + int direction = (flags & 1) ? DMA_TO_DEVICE : DMA_FROM_DEVICE; + return t2bce_dma_create_sq(dma, cq, name, el_count, direction, compl, userdata); +} +EXPORT_SYMBOL_GPL(t2bce_dma_create_sq_with_flags); + +void t2bce_dma_destroy_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq) +{ + if (!dma->is_being_removed && t2bce_dma_unregister_queue(dma, (u16) cq->qid)) + pr_err("t2bce_dma: CQ unregister failed"); + spin_lock(&dma->queues_lock); + dma->queues[cq->qid] = NULL; + spin_unlock(&dma->queues_lock); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + ida_simple_remove(&dma->queue_ida, (uint) cq->qid); +#else + ida_free(&dma->queue_ida, (uint) cq->qid); +#endif + t2bce_dma_free_cq(dma, cq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_destroy_cq); + +void t2bce_dma_destroy_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq) +{ + if (!dma->is_being_removed && t2bce_dma_unregister_queue(dma, (u16) sq->qid)) + pr_err("t2bce_dma: SQ unregister failed"); + spin_lock(&dma->queues_lock); + dma->queues[sq->qid] = NULL; + spin_unlock(&dma->queues_lock); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0) + ida_simple_remove(&dma->queue_ida, (uint) sq->qid); +#else + ida_free(&dma->queue_ida, (uint) sq->qid); +#endif + t2bce_dma_free_sq(dma, sq); +} +EXPORT_SYMBOL_GPL(t2bce_dma_destroy_sq); + +int t2bce_dma_flush_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq) +{ + return t2bce_dma_flush_queue(dma, (u16) sq->qid); +} +EXPORT_SYMBOL_GPL(t2bce_dma_flush_sq); + +static int __init t2bce_dma_module_init(void) +{ + pr_info("t2bce_dma: module initialized\n"); + return 0; +} + +static void __exit t2bce_dma_module_exit(void) +{ + pr_info("t2bce_dma: module exited\n"); +} + +module_init(t2bce_dma_module_init); +module_exit(t2bce_dma_module_exit); + +MODULE_AUTHOR("AndrĂ© Eikmeyer "); +MODULE_DESCRIPTION("Apple T2 BCE DMA queue engine"); +MODULE_VERSION("0.01"); +MODULE_LICENSE("GPL"); diff --git a/drivers/staging/t2bce/t2bce_vhci/Makefile b/drivers/staging/t2bce/t2bce_vhci/Makefile new file mode 100644 index 000000000..45e988609 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_vhci/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_T2BCE_VHCI) += t2bce_vhci.o +t2bce_vhci-y := vhci.o queue.o transfer.o + +ccflags-y += -DWITHOUT_NVME_PATCH +ccflags-y += -I$(src) +ccflags-y += -I$(src)/../t2bce_core/include diff --git a/drivers/staging/t2bce/t2bce_vhci/command.h b/drivers/staging/t2bce/t2bce_vhci/command.h new file mode 100644 index 000000000..6dc5fc94f --- /dev/null +++ b/drivers/staging/t2bce/t2bce_vhci/command.h @@ -0,0 +1,206 @@ +#ifndef BCE_VHCI_COMMAND_H +#define BCE_VHCI_COMMAND_H + +#include "queue.h" +#include +#include + +#define BCE_VHCI_CMD_TIMEOUT_SHORT msecs_to_jiffies(2000) +#define BCE_VHCI_CMD_TIMEOUT_LONG msecs_to_jiffies(30000) + +#define BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2 2 +#define BCE_VHCI_BULK_MAX_ACTIVE_URBS (1 << BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2) + +typedef u8 bce_vhci_port_t; +typedef u8 bce_vhci_device_t; + +enum bce_vhci_command { + BCE_VHCI_CMD_CONTROLLER_ENABLE = 1, + BCE_VHCI_CMD_CONTROLLER_DISABLE = 2, + BCE_VHCI_CMD_CONTROLLER_START = 3, + /* Reserved for future controller-level PM work; endpoint PM is used today. */ + BCE_VHCI_CMD_CONTROLLER_PAUSE = 4, + + BCE_VHCI_CMD_PORT_POWER_ON = 0x10, + BCE_VHCI_CMD_PORT_POWER_OFF = 0x11, + BCE_VHCI_CMD_PORT_RESUME = 0x12, + BCE_VHCI_CMD_PORT_SUSPEND = 0x13, + BCE_VHCI_CMD_PORT_RESET = 0x14, + BCE_VHCI_CMD_PORT_DISABLE = 0x15, + BCE_VHCI_CMD_PORT_STATUS = 0x16, + BCE_VHCI_EV_PORT_STATUS_CHANGE = 0x18, + + BCE_VHCI_CMD_DEVICE_CREATE = 0x30, + BCE_VHCI_CMD_DEVICE_DESTROY = 0x31, + + BCE_VHCI_CMD_ENDPOINT_CREATE = 0x40, + BCE_VHCI_CMD_ENDPOINT_DESTROY = 0x41, + BCE_VHCI_CMD_ENDPOINT_SET_STATE = 0x42, + BCE_VHCI_CMD_ENDPOINT_RESET = 0x44, + + /* Device to host only */ + BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE = 0x43, + BCE_VHCI_CMD_TRANSFER_REQUEST = 0x1000, + BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS = 0x1005 +}; + +enum bce_vhci_endpoint_state { + BCE_VHCI_ENDPOINT_ACTIVE = 0, + BCE_VHCI_ENDPOINT_PAUSED = 1, + BCE_VHCI_ENDPOINT_STALLED = 2 +}; + +static inline int bce_vhci_cmd_controller_enable(struct bce_vhci_command_queue *q, u8 busNum, u16 *portMask) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_ENABLE; + cmd.param1 = 0x7100u | busNum; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); + if (!status) + *portMask = (u16) res.param2; + return status; +} +static inline int bce_vhci_cmd_controller_disable(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_controller_start(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_START; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_controller_pause(struct bce_vhci_command_queue *q) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_CONTROLLER_PAUSE; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} + +static inline int bce_vhci_cmd_port_power_on(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_POWER_ON; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_power_off(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_POWER_OFF; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_resume(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_RESUME; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_port_suspend(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_SUSPEND; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} +static inline int bce_vhci_cmd_port_reset(struct bce_vhci_command_queue *q, bce_vhci_port_t port, u32 timeout) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_RESET; + cmd.param1 = port; + cmd.param2 = timeout; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_disable(struct bce_vhci_command_queue *q, bce_vhci_port_t port) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_DISABLE; + cmd.param1 = port; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_port_status(struct bce_vhci_command_queue *q, bce_vhci_port_t port, + u32 clearFlags, u32 *resStatus) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_PORT_STATUS; + cmd.param1 = port; + cmd.param2 = clearFlags & 0x560000; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (status >= 0) + *resStatus = (u32) res.param2; + return status; +} + +static inline int bce_vhci_cmd_device_create(struct bce_vhci_command_queue *q, bce_vhci_port_t port, + bce_vhci_device_t *dev) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_DEVICE_CREATE; + cmd.param1 = port; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (!status) + *dev = (bce_vhci_device_t) res.param2; + return status; +} +static inline int bce_vhci_cmd_device_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY; + cmd.param1 = dev; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG); +} + +static inline int bce_vhci_cmd_endpoint_create(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, + struct usb_endpoint_descriptor *desc) +{ + struct bce_vhci_message cmd, res; + int endpoint_type = usb_endpoint_type(desc); + int maxp = usb_endpoint_maxp(desc); + int maxp_burst = usb_endpoint_maxp_mult(desc) * maxp; + u8 max_active_requests_pow2 = 0; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_CREATE; + cmd.param1 = dev | ((desc->bEndpointAddress & 0x8Fu) << 8); + if (endpoint_type == USB_ENDPOINT_XFER_BULK) + max_active_requests_pow2 = BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2; + cmd.param2 = endpoint_type | ((max_active_requests_pow2 & 0xf) << 4) | (maxp << 16) | ((u64) maxp_burst << 32); + if (endpoint_type == USB_ENDPOINT_XFER_INT) + cmd.param2 |= (desc->bInterval - 1) << 8; + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_endpoint_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_DESTROY; + cmd.param1 = dev | (endpoint << 8); + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} +static inline int bce_vhci_cmd_endpoint_set_state(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint, + enum bce_vhci_endpoint_state newState, enum bce_vhci_endpoint_state *retState) +{ + int status; + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_SET_STATE; + cmd.param1 = dev | (endpoint << 8); + cmd.param2 = (u64) newState; + status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); + if (status != BCE_VHCI_INTERNAL_ERROR && status != BCE_VHCI_NO_POWER) + *retState = (enum bce_vhci_endpoint_state) res.param2; + return status; +} +static inline int bce_vhci_cmd_endpoint_reset(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint) +{ + struct bce_vhci_message cmd, res; + cmd.cmd = BCE_VHCI_CMD_ENDPOINT_RESET; + cmd.param1 = dev | (endpoint << 8); + return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT); +} + + +#endif //BCE_VHCI_COMMAND_H diff --git a/drivers/staging/t2bce/t2bce_vhci/queue.c b/drivers/staging/t2bce/t2bce_vhci/queue.c new file mode 100644 index 000000000..63779fc92 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_vhci/queue.c @@ -0,0 +1,269 @@ +#include "queue.h" +#include "vhci.h" + +#include + + +static void bce_vhci_message_queue_completion(struct t2bce_core_queue_sq *sq); + +int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name) +{ + int status; + ret->cq = t2bce_core_create_cq(vhci->client, VHCI_EVENT_QUEUE_EL_COUNT); + if (!ret->cq) + return -EINVAL; + ret->sq = t2bce_core_create_sq(vhci->client, ret->cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_TO_DEVICE, + bce_vhci_message_queue_completion, ret); + if (!ret->sq) { + status = -EINVAL; + goto fail_cq; + } + ret->data = dma_alloc_coherent(t2bce_core_client_dma_dev(vhci->client), + sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + &ret->dma_addr, GFP_KERNEL); + if (!ret->data) { + status = -EINVAL; + goto fail_sq; + } + return 0; + +fail_sq: + t2bce_core_destroy_sq(vhci->client, ret->sq); + ret->sq = NULL; +fail_cq: + t2bce_core_destroy_cq(vhci->client, ret->cq); + ret->cq = NULL; + return status; +} + +void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q) +{ + if (!q->cq) + return; + dma_free_coherent(t2bce_core_client_dma_dev(vhci->client), + sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + q->data, q->dma_addr); + t2bce_core_destroy_sq(vhci->client, q->sq); + t2bce_core_destroy_cq(vhci->client, q->cq); +} + +void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req) +{ + int sidx; + sidx = t2bce_core_queue_sq_tail(q->sq); + pr_debug("t2bce_vhci: Send message: %x s=%x p1=%x p2=%llx\n", req->cmd, req->status, req->param1, req->param2); + q->data[sidx] = *req; + t2bce_core_set_next_submission_single(q->sq, q->dma_addr + sizeof(struct bce_vhci_message) * sidx, + sizeof(struct bce_vhci_message)); + t2bce_core_submit_to_device(q->sq); +} + +static void bce_vhci_message_queue_completion(struct t2bce_core_queue_sq *sq) +{ + while (t2bce_core_next_completion(sq)) + t2bce_core_notify_submission_complete(sq); +} + + + +static void bce_vhci_event_queue_completion(struct t2bce_core_queue_sq *sq); + +int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + t2bce_core_sq_completion compl) +{ + ret->vhci = vhci; + + ret->sq = t2bce_core_create_sq(vhci->client, vhci->ev_cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_FROM_DEVICE, compl, ret); + if (!ret->sq) + return -EINVAL; + ret->data = dma_alloc_coherent(t2bce_core_client_dma_dev(vhci->client), + sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + &ret->dma_addr, GFP_KERNEL); + if (!ret->data) { + t2bce_core_destroy_sq(vhci->client, ret->sq); + ret->sq = NULL; + return -EINVAL; + } + + init_completion(&ret->queue_empty_completion); + bce_vhci_event_queue_submit_pending(ret, VHCI_EVENT_PENDING_COUNT); + return 0; +} + +int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_vhci_event_queue_callback cb) +{ + ret->cb = cb; + return __bce_vhci_event_queue_create(vhci, ret, name, bce_vhci_event_queue_completion); +} + +void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q) +{ + if (!q->sq) + return; + dma_free_coherent(t2bce_core_client_dma_dev(vhci->client), + sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT, + q->data, q->dma_addr); + t2bce_core_destroy_sq(vhci->client, q->sq); +} + +static void bce_vhci_event_queue_completion(struct t2bce_core_queue_sq *sq) +{ + struct t2bce_core_sq_completion_data *cd; + struct bce_vhci_event_queue *ev = t2bce_core_queue_sq_userdata(sq); + struct bce_vhci_message *msg; + size_t cnt = 0; + + while ((cd = t2bce_core_next_completion(sq))) { + if (cd->status == T2BCE_COMPLETION_ABORTED) { + t2bce_core_notify_submission_complete(sq); + continue; + } + msg = &ev->data[t2bce_core_queue_sq_head(sq)]; + pr_debug("t2bce_vhci: Got event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); + ev->cb(ev, msg); + + t2bce_core_notify_submission_complete(sq); + ++cnt; + } + bce_vhci_event_queue_submit_pending(ev, cnt); + if (t2bce_core_queue_sq_available(sq) == t2bce_core_queue_sq_capacity(sq) - 1) + complete(&ev->queue_empty_completion); +} + +void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count) +{ + int idx; + while (count--) { + if (t2bce_core_reserve_submission(q->sq, NULL)) { + pr_err("t2bce_vhci: Failed to reserve an event queue submission\n"); + break; + } + idx = t2bce_core_queue_sq_tail(q->sq); + t2bce_core_set_next_submission_single(q->sq, + q->dma_addr + idx * sizeof(struct bce_vhci_message), sizeof(struct bce_vhci_message)); + } + t2bce_core_submit_to_device(q->sq); +} + +void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q) +{ + unsigned long timeout; + reinit_completion(&q->queue_empty_completion); + if (t2bce_core_flush_queue(q->vhci->client, q->sq)) + pr_warn("t2bce_vhci: failed to flush event queue\n"); + timeout = msecs_to_jiffies(5000); + while (t2bce_core_queue_sq_available(q->sq) != t2bce_core_queue_sq_capacity(q->sq) - 1) { + timeout = wait_for_completion_timeout(&q->queue_empty_completion, timeout); + if (timeout == 0) { + pr_err("t2bce_vhci: waiting for queue to be flushed timed out\n"); + break; + } + } +} + +void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q) +{ + if (t2bce_core_queue_sq_available(q->sq) != t2bce_core_queue_sq_capacity(q->sq) - 1) { + pr_err("t2bce_vhci: resume of a queue with pending submissions\n"); + return; + } + bce_vhci_event_queue_submit_pending(q, VHCI_EVENT_PENDING_COUNT); +} + +void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq) +{ + ret->mq = mq; + ret->completion.result = NULL; + init_completion(&ret->completion.completion); + spin_lock_init(&ret->completion_lock); + mutex_init(&ret->mutex); +} + +void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq) +{ + spin_lock(&cq->completion_lock); + if (cq->completion.result) { + memset(cq->completion.result, 0, sizeof(struct bce_vhci_message)); + cq->completion.result->status = BCE_VHCI_ABORT; + complete(&cq->completion.completion); + cq->completion.result = NULL; + } + spin_unlock(&cq->completion_lock); + mutex_lock(&cq->mutex); + mutex_unlock(&cq->mutex); + mutex_destroy(&cq->mutex); +} + +void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg) +{ + struct bce_vhci_command_queue_completion *c = &cq->completion; + + spin_lock(&cq->completion_lock); + if (c->result) { + *c->result = *msg; + complete(&c->completion); + c->result = NULL; + } + spin_unlock(&cq->completion_lock); +} + +static int __bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout) +{ + int status; + struct bce_vhci_command_queue_completion *c; + struct bce_vhci_message creq; + c = &cq->completion; + + if ((status = t2bce_core_reserve_submission(cq->mq->sq, &timeout))) + return status; + + spin_lock(&cq->completion_lock); + c->result = res; + reinit_completion(&c->completion); + spin_unlock(&cq->completion_lock); + + bce_vhci_message_queue_write(cq->mq, req); + + if (!wait_for_completion_timeout(&c->completion, timeout)) { + /* Ask bridgeOS to cancel the command before reporting timeout. */ + pr_debug("t2bce_vhci: command timed out req=%x\n", req->cmd); + if ((status = t2bce_core_reserve_submission(cq->mq->sq, &timeout))) + return status; + + creq = *req; + creq.cmd |= 0x4000; + bce_vhci_message_queue_write(cq->mq, &creq); + + if (!wait_for_completion_timeout(&c->completion, 1000)) { + pr_err("t2bce_vhci: Possible desync, cmd cancel timed out\n"); + + spin_lock(&cq->completion_lock); + c->result = NULL; + spin_unlock(&cq->completion_lock); + return -ETIMEDOUT; + } + if ((res->cmd & ~0x8000) == creq.cmd) + return -ETIMEDOUT; + /* The original reply arrived after the cancellation was queued. */ + } + + if ((res->cmd & ~0x8000) != req->cmd) { + pr_err("t2bce_vhci: Possible desync, cmd reply mismatch req=%x, res=%x\n", req->cmd, res->cmd); + return -EIO; + } + if (res->status == BCE_VHCI_SUCCESS) + return 0; + return res->status; +} + +int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout) +{ + int status; + mutex_lock(&cq->mutex); + status = __bce_vhci_command_queue_execute(cq, req, res, timeout); + mutex_unlock(&cq->mutex); + return status; +} diff --git a/drivers/staging/t2bce/t2bce_vhci/queue.h b/drivers/staging/t2bce/t2bce_vhci/queue.h new file mode 100644 index 000000000..044eb2971 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_vhci/queue.h @@ -0,0 +1,76 @@ +#ifndef BCE_VHCI_QUEUE_H +#define BCE_VHCI_QUEUE_H + +#include +#include "t2bce_core_transport.h" + +#define VHCI_EVENT_QUEUE_EL_COUNT 256 +#define VHCI_EVENT_PENDING_COUNT 32 + +struct bce_vhci; +struct bce_vhci_event_queue; + +enum bce_vhci_message_status { + BCE_VHCI_SUCCESS = 1, + BCE_VHCI_ERROR = 2, + BCE_VHCI_USB_PIPE_STALL = 3, + BCE_VHCI_ABORT = 4, + BCE_VHCI_BAD_ARGUMENT = 5, + BCE_VHCI_OVERRUN = 6, + BCE_VHCI_INTERNAL_ERROR = 7, + BCE_VHCI_NO_POWER = 8, + BCE_VHCI_UNSUPPORTED = 9 +}; +struct bce_vhci_message { + u16 cmd; + u16 status; // bce_vhci_message_status + u32 param1; + u64 param2; +}; + +struct bce_vhci_message_queue { + struct t2bce_core_queue_cq *cq; + struct t2bce_core_queue_sq *sq; + struct bce_vhci_message *data; + dma_addr_t dma_addr; +}; +typedef void (*bce_vhci_event_queue_callback)(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); +struct bce_vhci_event_queue { + struct bce_vhci *vhci; + struct t2bce_core_queue_sq *sq; + struct bce_vhci_message *data; + dma_addr_t dma_addr; + bce_vhci_event_queue_callback cb; + struct completion queue_empty_completion; +}; +struct bce_vhci_command_queue_completion { + struct bce_vhci_message *result; + struct completion completion; +}; +struct bce_vhci_command_queue { + struct bce_vhci_message_queue *mq; + struct bce_vhci_command_queue_completion completion; + struct spinlock completion_lock; + struct mutex mutex; +}; + +int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name); +void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q); +void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req); + +int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + t2bce_core_sq_completion compl); +int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name, + bce_vhci_event_queue_callback cb); +void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q); +void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count); +void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q); +void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q); + +void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq); +void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq); +int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req, + struct bce_vhci_message *res, unsigned long timeout); +void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg); + +#endif //BCE_VHCI_QUEUE_H diff --git a/drivers/staging/t2bce/t2bce_vhci/transfer.c b/drivers/staging/t2bce/t2bce_vhci/transfer.c new file mode 100644 index 000000000..1c9643e13 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_vhci/transfer.c @@ -0,0 +1,1261 @@ +#include "transfer.h" +#include "vhci.h" +#include +#include +#include + +static void bce_vhci_transfer_queue_completion(struct t2bce_core_queue_sq *sq); +static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q); +static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q); + +static int bce_vhci_urb_init(struct bce_vhci_urb *vurb); +static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg); +static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c); +static void bce_vhci_urb_complete(struct bce_vhci_urb *urb, int status); +static void bce_vhci_urb_free(struct bce_vhci_urb *urb); + +static void bce_vhci_transfer_queue_reset_w(struct work_struct *work); +static void bce_vhci_transfer_queue_resume_w(struct work_struct *work); +static void bce_vhci_transfer_queue_cancel_w(struct work_struct *work); + +void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, + struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir) +{ + char name[0x21]; + INIT_LIST_HEAD(&q->evq); + INIT_LIST_HEAD(&q->giveback_urb_list); + INIT_LIST_HEAD(&q->cancel_urb_list); + spin_lock_init(&q->urb_lock); + mutex_init(&q->pause_lock); + init_waitqueue_head(&q->sq_out_wait_queue); + atomic_set(&q->sq_out_pending, 0); + q->vhci = vhci; + q->endp = endp; + q->dev_addr = dev_addr; + q->endp_addr = (u8) (endp->desc.bEndpointAddress & 0x8F); + q->state = BCE_VHCI_ENDPOINT_ACTIVE; + q->active = true; + q->stalled = false; + q->paused_by = 0; + q->max_active_requests = 1; + if (usb_endpoint_type(&endp->desc) == USB_ENDPOINT_XFER_BULK) + q->max_active_requests = BCE_VHCI_BULK_MAX_ACTIVE_URBS; + q->remaining_active_requests = q->max_active_requests; + q->cq = t2bce_core_create_cq(vhci->client, 0x100); + INIT_WORK(&q->w_reset, bce_vhci_transfer_queue_reset_w); + INIT_WORK(&q->w_resume, bce_vhci_transfer_queue_resume_w); + INIT_WORK(&q->w_cancel, bce_vhci_transfer_queue_cancel_w); + q->sq_in = NULL; + if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) { + snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, 0x80 | usb_endpoint_num(&endp->desc)); + q->sq_in = t2bce_core_create_sq(vhci->client, q->cq, name, 0x100, DMA_FROM_DEVICE, + bce_vhci_transfer_queue_completion, q); + } + q->sq_out = NULL; + if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) { + snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, usb_endpoint_num(&endp->desc)); + q->sq_out = t2bce_core_create_sq(vhci->client, q->cq, name, 0x100, DMA_TO_DEVICE, + bce_vhci_transfer_queue_completion, q); + } +} + +void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q) +{ + cancel_work_sync(&q->w_resume); + cancel_work_sync(&q->w_reset); + cancel_work_sync(&q->w_cancel); + bce_vhci_transfer_queue_giveback(q); + bce_vhci_transfer_queue_remove_pending(q); + if (q->sq_in) + t2bce_core_destroy_sq(vhci->client, q->sq_in); + if (q->sq_out) + t2bce_core_destroy_sq(vhci->client, q->sq_out); + t2bce_core_destroy_cq(vhci->client, q->cq); +} + +static inline bool bce_vhci_transfer_queue_can_init_urb(struct bce_vhci_transfer_queue *q) +{ + return q->remaining_active_requests > 0; +} + +static bool bce_vhci_transfer_queue_defer_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) +{ + struct bce_vhci_list_message *lm; + lm = kmalloc(sizeof(struct bce_vhci_list_message), GFP_ATOMIC); + if (!lm) { + pr_warn("t2bce_vhci: [%02x] failed to defer transfer queue event cmd=%x status=%x\n", + q->endp_addr, msg->cmd, msg->status); + return false; + } + INIT_LIST_HEAD(&lm->list); + lm->msg = *msg; + list_add_tail(&lm->list, &q->evq); + return true; +} + +static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + struct urb *urb; + spin_lock_irqsave(&q->urb_lock, flags); + while (!list_empty(&q->giveback_urb_list)) { + urb = list_first_entry(&q->giveback_urb_list, struct urb, urb_list); + list_del(&urb->urb_list); + + spin_unlock_irqrestore(&q->urb_lock, flags); + usb_hcd_giveback_urb(q->vhci->hcd, urb, urb->status); + spin_lock_irqsave(&q->urb_lock, flags); + } + spin_unlock_irqrestore(&q->urb_lock, flags); +} + +static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q); + +static int bce_vhci_transfer_queue_port(struct bce_vhci_transfer_queue *q) +{ + int port; + + for (port = 1; port <= q->vhci->port_count && + port < ARRAY_SIZE(q->vhci->port_to_device); port++) { + if (q->vhci->port_to_device[port] == q->dev_addr) + return port; + } + + return 0; +} + +static bool bce_vhci_transfer_queue_is_ep0(struct bce_vhci_transfer_queue *q) +{ + return q->endp_addr == 0x00; +} + +static size_t bce_vhci_transfer_queue_event_count(struct bce_vhci_transfer_queue *q) +{ + struct bce_vhci_list_message *lm; + size_t count = 0; + + list_for_each_entry(lm, &q->evq, list) + count++; + + return count; +} + +static void bce_vhci_transfer_queue_log_event(struct bce_vhci_transfer_queue *q, + struct bce_vhci_message *msg, const char *stage) +{ + if (!bce_vhci_transfer_queue_is_ep0(q)) + return; + + pr_debug("t2bce_vhci: EP0 event %s dev=%u port=%d cmd=%x msg_status=%x p1=%x p2=%llx active=%u paused_by=%x stalled=%u qstate=%u remaining=%u evq=%zu urb_empty=%d\n", + stage, q->dev_addr, bce_vhci_transfer_queue_port(q), + msg->cmd, msg->status, msg->param1, msg->param2, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests, + bce_vhci_transfer_queue_event_count(q), + list_empty(&q->endp->urb_list)); +} + +static void bce_vhci_transfer_queue_log_completion(struct bce_vhci_transfer_queue *q, + struct t2bce_core_queue_sq *sq, struct t2bce_core_sq_completion_data *c, const char *stage) +{ + const char *sq_name = "unknown"; + + if (!bce_vhci_transfer_queue_is_ep0(q)) + return; + + if (sq == q->sq_in) + sq_name = "in"; + else if (sq == q->sq_out) + sq_name = "out"; + + pr_debug("t2bce_vhci: EP0 completion %s dev=%u port=%d sq=%s cstatus=%x data_size=%llu sq_out_pending=%d active=%u paused_by=%x stalled=%u qstate=%u remaining=%u evq=%zu urb_empty=%d\n", + stage, q->dev_addr, bce_vhci_transfer_queue_port(q), + sq_name, c->status, c->data_size, + atomic_read(&q->sq_out_pending), q->active, q->paused_by, + q->stalled, q->state, q->remaining_active_requests, + bce_vhci_transfer_queue_event_count(q), + list_empty(&q->endp->urb_list)); +} + +static void bce_vhci_urb_log_control(struct bce_vhci_urb *urb, const char *stage) +{ + struct bce_vhci_transfer_queue *q = urb->q; + struct usb_ctrlrequest *ctrl; + + if (!urb->is_control || !bce_vhci_transfer_queue_is_ep0(q)) + return; + + ctrl = (struct usb_ctrlrequest *) urb->urb->setup_packet; + if (!ctrl) { + pr_debug("t2bce_vhci: EP0 control %s dev=%u port=%d urb_state=%u no-setup xfer_len=%u actual=%u send=%u recv=%u status=%d active=%u paused_by=%x stalled=%u qstate=%u remaining=%u urb_reject=%d\n", + stage, q->dev_addr, bce_vhci_transfer_queue_port(q), + urb->state, urb->urb->transfer_buffer_length, + urb->urb->actual_length, urb->send_offset, + urb->receive_offset, urb->received_status, q->active, + q->paused_by, q->stalled, q->state, + q->remaining_active_requests, atomic_read(&urb->urb->reject)); + return; + } + + /* Per-stage EP0 control-transfer trace. + * Keep this at pr_debug; enable dynamic debug when diagnosing + * setup/data/status ordering or endpoint recovery issues. + */ + pr_debug("t2bce_vhci: EP0 control %s dev=%u port=%d urb_state=%u dir=%s reqtype=%02x req=%02x value=%04x index=%04x wlen=%u xfer_len=%u actual=%u send=%u recv=%u status=%d active=%u paused_by=%x stalled=%u qstate=%u remaining=%u urb_reject=%d\n", + stage, q->dev_addr, bce_vhci_transfer_queue_port(q), + urb->state, usb_urb_dir_in(urb->urb) ? "in" : "out", + ctrl->bRequestType, ctrl->bRequest, + le16_to_cpu(ctrl->wValue), le16_to_cpu(ctrl->wIndex), + le16_to_cpu(ctrl->wLength), urb->urb->transfer_buffer_length, + urb->urb->actual_length, urb->send_offset, urb->receive_offset, + urb->received_status, q->active, q->paused_by, q->stalled, + q->state, q->remaining_active_requests, + atomic_read(&urb->urb->reject)); +} + +static void bce_vhci_transfer_queue_deliver_pending(struct bce_vhci_transfer_queue *q) +{ + struct urb *urb; + struct bce_vhci_list_message *lm; + + while (!list_empty(&q->endp->urb_list) && !list_empty(&q->evq)) { + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + + lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); + bce_vhci_transfer_queue_log_event(q, &lm->msg, "deferred-deliver"); + if (bce_vhci_urb_update(urb->hcpriv, &lm->msg) == -EAGAIN) { + bce_vhci_transfer_queue_log_event(q, &lm->msg, "deferred-still-waiting"); + break; + } + bce_vhci_transfer_queue_log_event(q, &lm->msg, "deferred-consumed"); + list_del(&lm->list); + kfree(lm); + } + + /* some of the URBs could have been completed, so initialize more URBs if possible */ + bce_vhci_transfer_queue_init_pending_urbs(q); +} + +static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + struct bce_vhci_list_message *lm; + spin_lock_irqsave(&q->urb_lock, flags); + if (bce_vhci_transfer_queue_is_ep0(q) && !list_empty(&q->evq)) + pr_debug("t2bce_vhci: EP0 event remove-pending dev=%u port=%d evq=%zu active=%u paused_by=%x stalled=%u qstate=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), + bce_vhci_transfer_queue_event_count(q), q->active, + q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + while (!list_empty(&q->evq)) { + lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list); + list_del(&lm->list); + kfree(lm); + } + spin_unlock_irqrestore(&q->urb_lock, flags); +} + +void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg) +{ + unsigned long flags; + struct bce_vhci_urb *turb; + struct urb *urb; + spin_lock_irqsave(&q->urb_lock, flags); + bce_vhci_transfer_queue_log_event(q, msg, "incoming"); + /* Paused queues may still complete in-flight work but must not deliver new work. */ + if (!q->active) { + if (bce_vhci_transfer_queue_is_ep0(q) && + msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && + (q->paused_by & BCE_VHCI_PAUSE_INTERNAL_WQ)) { + bce_vhci_transfer_queue_log_event(q, msg, "defer-inactive-internal-pause"); + if (!bce_vhci_transfer_queue_defer_event(q, msg)) + bce_vhci_transfer_queue_log_event(q, msg, "defer-inactive-failed"); + } else { + bce_vhci_transfer_queue_log_event(q, msg, "drop-inactive"); + } + goto complete; + } + bce_vhci_transfer_queue_deliver_pending(q); + + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && + (!list_empty(&q->evq) || list_empty(&q->endp->urb_list))) { + bce_vhci_transfer_queue_log_event(q, msg, "defer-no-urb-or-backlog"); + if (!bce_vhci_transfer_queue_defer_event(q, msg)) + bce_vhci_transfer_queue_log_event(q, msg, "defer-no-urb-or-backlog-failed"); + goto complete; + } + if (list_empty(&q->endp->urb_list)) { + bce_vhci_transfer_queue_log_event(q, msg, "unexpected-empty"); + pr_err("t2bce_vhci: [%02x] Unexpected transfer queue event\n", q->endp_addr); + goto complete; + } + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + turb = urb->hcpriv; + if (bce_vhci_urb_update(turb, msg) == -EAGAIN) { + bce_vhci_transfer_queue_log_event(q, msg, "defer-eagain"); + if (!bce_vhci_transfer_queue_defer_event(q, msg)) + bce_vhci_transfer_queue_log_event(q, msg, "defer-eagain-failed"); + } else { + bce_vhci_transfer_queue_log_event(q, msg, "consumed"); + bce_vhci_transfer_queue_init_pending_urbs(q); + } + +complete: + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_giveback(q); +} + +static void bce_vhci_transfer_queue_completion(struct t2bce_core_queue_sq *sq) +{ + unsigned long flags; + struct t2bce_core_sq_completion_data *c; + struct urb *urb; + struct bce_vhci_transfer_queue *q = t2bce_core_queue_sq_userdata(sq); + bool is_sq_out = (sq == q->sq_out); + spin_lock_irqsave(&q->urb_lock, flags); + while ((c = t2bce_core_next_completion(sq))) { + if (c->status == T2BCE_COMPLETION_ABORTED) { /* We flushed the queue */ + bce_vhci_transfer_queue_log_completion(q, sq, c, "aborted"); + pr_debug("t2bce_vhci: [%02x] Got an abort completion\n", q->endp_addr); + if (is_sq_out && atomic_dec_if_positive(&q->sq_out_pending) == 0) + wake_up(&q->sq_out_wait_queue); + t2bce_core_notify_submission_complete(sq); + continue; + } + if (list_empty(&q->endp->urb_list)) { + bce_vhci_transfer_queue_log_completion(q, sq, c, "empty"); + pr_err("t2bce_vhci: [%02x] Got a completion while no requests are pending\n", q->endp_addr); + if (is_sq_out && atomic_dec_if_positive(&q->sq_out_pending) == 0) + wake_up(&q->sq_out_wait_queue); + continue; + } + bce_vhci_transfer_queue_log_completion(q, sq, c, "incoming"); + pr_debug("t2bce_vhci: [%02x] Got a transfer queue completion\n", q->endp_addr); + urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list); + bce_vhci_urb_transfer_completion(urb->hcpriv, c); + if (is_sq_out && atomic_dec_if_positive(&q->sq_out_pending) == 0) + wake_up(&q->sq_out_wait_queue); + t2bce_core_notify_submission_complete(sq); + } + bce_vhci_transfer_queue_deliver_pending(q); + spin_unlock_irqrestore(&q->urb_lock, flags); + bce_vhci_transfer_queue_giveback(q); +} + +int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + int pending; + long timeout; + spin_lock_irqsave(&q->urb_lock, flags); + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + if (q->sq_out) { + /* Let pending OUT submissions drain before the endpoint is paused. */ + pending = atomic_read(&q->sq_out_pending); + if (pending > 0) { + timeout = wait_event_timeout(q->sq_out_wait_queue, + atomic_read(&q->sq_out_pending) == 0, + msecs_to_jiffies(2000)); + if (!timeout && atomic_read(&q->sq_out_pending) > 0) + pr_warn("t2bce_vhci: [%02x] pause timeout waiting for %d outputs\n", + q->endp_addr, atomic_read(&q->sq_out_pending)); + } + } + bce_vhci_transfer_queue_remove_pending(q); + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_PAUSED, &q->state))) + return status; + if (q->state != BCE_VHCI_ENDPOINT_PAUSED) + return -EINVAL; + if (q->sq_in) + t2bce_core_flush_queue(q->vhci->client, q->sq_in); + if (q->sq_out) + t2bce_core_flush_queue(q->vhci->client, q->sq_out); + return 0; +} + +static void bce_vhci_urb_resume(struct bce_vhci_urb *urb); + +int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + struct urb *urb, *urbt; + struct bce_vhci_urb *vurb; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + if (bce_vhci_transfer_queue_is_ep0(q)) + pr_debug("t2bce_vhci: tq resume begin dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_ACTIVE, &q->state))) { + pr_warn("t2bce_vhci: tq resume set-active failed dev=%u port=%d ep=%02x status=%d ret_state=%u active=%u paused_by=%x stalled=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + status, q->state, q->active, q->paused_by, q->stalled); + return status; + } + if (q->state != BCE_VHCI_ENDPOINT_ACTIVE) { + pr_warn("t2bce_vhci: tq resume set-active returned non-active dev=%u port=%d ep=%02x ret_state=%u active=%u paused_by=%x stalled=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->state, q->active, q->paused_by, q->stalled); + return -EINVAL; + } + spin_lock_irqsave(&q->urb_lock, flags); + q->active = true; + list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { + vurb = urb->hcpriv; + if (vurb->state == BCE_VHCI_URB_INIT_PENDING) { + if (!bce_vhci_transfer_queue_can_init_urb(q)) + break; + bce_vhci_urb_init(vurb); + } else { + bce_vhci_urb_resume(vurb); + } + } + bce_vhci_transfer_queue_deliver_pending(q); + spin_unlock_irqrestore(&q->urb_lock, flags); + if (bce_vhci_transfer_queue_is_ep0(q)) + pr_debug("t2bce_vhci: tq resume done dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + return 0; +} + +int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) +{ + int ret = 0; + u32 old_paused_by; + + mutex_lock(&q->pause_lock); + old_paused_by = q->paused_by; + if ((q->paused_by & src) != src) { + if (!q->paused_by) + ret = bce_vhci_transfer_queue_do_pause(q); + if (!ret) + q->paused_by |= src; + } + if (bce_vhci_transfer_queue_is_ep0(q) || ret) + pr_debug("t2bce_vhci: tq pause dev=%u port=%d ep=%02x src=%x ret=%d paused_by=%x->%x active=%u stalled=%u state=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + src, ret, old_paused_by, q->paused_by, q->active, + q->stalled, q->state); + mutex_unlock(&q->pause_lock); + return ret; +} + +int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src) +{ + int ret = 0; + u32 old_paused_by; + bool did_resume = false; + bool cleared_shutdown = false; + + mutex_lock(&q->pause_lock); + old_paused_by = q->paused_by; + /* + * EP0 queues can carry the old shutdown pause across preserved-state + * resume. System resume owns that transition, so clear the shutdown owner + * here before deciding whether the queue needs a firmware resume. + */ + if (src == BCE_VHCI_PAUSE_SUSPEND && + q->endp_addr == 0x00 && + q->paused_by == (BCE_VHCI_PAUSE_SHUTDOWN | BCE_VHCI_PAUSE_SUSPEND)) { + q->paused_by &= ~BCE_VHCI_PAUSE_SHUTDOWN; + cleared_shutdown = true; + } + if (q->paused_by & src) { + if (!(q->paused_by & ~src)) { + ret = bce_vhci_transfer_queue_do_resume(q); + did_resume = true; + } + if (!ret) + q->paused_by &= ~src; + } + if (src == BCE_VHCI_PAUSE_SUSPEND) + pr_debug("t2bce_vhci: system resume dev=%u ep=%02x src=%x paused_by=%x->%x firmware_resume=%u cleared_shutdown=%u ret=%d state=%x active=%u\n", + q->dev_addr, q->endp_addr, src, old_paused_by, q->paused_by, + did_resume, cleared_shutdown, ret, q->state, q->active); + else if (q->endp_addr == 0x00) + pr_debug("t2bce_vhci: EP0 resume dev=%u src=%x paused_by=%x->%x did_resume=%u ret=%d state=%x active=%u\n", + q->dev_addr, src, old_paused_by, q->paused_by, did_resume, + ret, q->state, q->active); + mutex_unlock(&q->pause_lock); + return ret; +} + +int bce_vhci_transfer_queue_suspend_pause(struct bce_vhci_transfer_queue *q) +{ + unsigned long flags; + int status; + u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F); + int pending; + long timeout; + int ret = 0; + u32 old_paused_by; + + mutex_lock(&q->pause_lock); + old_paused_by = q->paused_by; + if ((q->paused_by & BCE_VHCI_PAUSE_SUSPEND) == BCE_VHCI_PAUSE_SUSPEND) { + pr_debug("t2bce_vhci: system suspend dev=%u ep=%02x path=already-suspend paused_by=%x state=%x active=%u\n", + q->dev_addr, q->endp_addr, q->paused_by, q->state, q->active); + goto out; + } + if (q->paused_by) { + /* + * Another owner already paused the endpoint. In that case system + * suspend only records its own ownership bit and must not send a + * second firmware pause for the same queue. + */ + q->paused_by |= BCE_VHCI_PAUSE_SUSPEND; + pr_debug("t2bce_vhci: system suspend dev=%u ep=%02x path=mark-only paused_by=%x->%x state=%x active=%u\n", + q->dev_addr, q->endp_addr, old_paused_by, q->paused_by, + q->state, q->active); + goto out; + } + + spin_lock_irqsave(&q->urb_lock, flags); + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + if (q->sq_out) { + pending = atomic_read(&q->sq_out_pending); + if (pending > 0) { + timeout = wait_event_timeout(q->sq_out_wait_queue, + atomic_read(&q->sq_out_pending) == 0, + msecs_to_jiffies(2000)); + if (!timeout && atomic_read(&q->sq_out_pending) > 0) + pr_warn("t2bce_vhci: [%02x] pause timeout waiting for %d outputs\n", + q->endp_addr, atomic_read(&q->sq_out_pending)); + } + } + /* + * System suspend is a preserved-state pause, not endpoint teardown. + * Deferred TRANSFER_REQUEST events already emitted by the T2 still belong + * to the same endpoint after resume; dropping them can make the first + * post-resume EP0 control URB wait until usbcore's timeout path. + */ + if ((status = bce_vhci_cmd_endpoint_set_state( + &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_PAUSED, &q->state))) { + ret = status; + goto out; + } + if (q->state != BCE_VHCI_ENDPOINT_PAUSED) { + ret = -EINVAL; + goto out; + } + q->paused_by |= BCE_VHCI_PAUSE_SUSPEND; + pr_debug("t2bce_vhci: system suspend dev=%u ep=%02x path=firmware-pause paused_by=%x->%x ret=%d state=%x active=%u\n", + q->dev_addr, q->endp_addr, old_paused_by, q->paused_by, + ret, q->state, q->active); + +out: + mutex_unlock(&q->pause_lock); + return ret; +} + +static void bce_vhci_transfer_queue_reset_w(struct work_struct *work) +{ + unsigned long flags; + struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_reset); + + mutex_lock(&q->pause_lock); + spin_lock_irqsave(&q->urb_lock, flags); + pr_debug("t2bce_vhci: tq reset start dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + if (!q->stalled) { + spin_unlock_irqrestore(&q->urb_lock, flags); + mutex_unlock(&q->pause_lock); + pr_debug("t2bce_vhci: tq reset skipped dev=%u port=%d ep=%02x stalled=0 active=%u paused_by=%x state=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->state); + return; + } + q->active = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + q->paused_by |= BCE_VHCI_PAUSE_INTERNAL_WQ; + bce_vhci_transfer_queue_remove_pending(q); + if (q->sq_in) + t2bce_core_flush_queue(q->vhci->client, q->sq_in); + if (q->sq_out) + t2bce_core_flush_queue(q->vhci->client, q->sq_out); + pr_debug("t2bce_vhci: tq reset command dev=%u port=%d ep=%02x\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr); + bce_vhci_cmd_endpoint_reset(&q->vhci->cq, q->dev_addr, (u8) (q->endp->desc.bEndpointAddress & 0x8F)); + spin_lock_irqsave(&q->urb_lock, flags); + q->stalled = false; + spin_unlock_irqrestore(&q->urb_lock, flags); + mutex_unlock(&q->pause_lock); + bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + pr_debug("t2bce_vhci: tq reset done dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); +} + +static void bce_vhci_transfer_queue_resume_w(struct work_struct *work) +{ + int ret; + struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_resume); + + pr_debug("t2bce_vhci: tq async resume start dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); + ret = bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + pr_debug("t2bce_vhci: tq async resume done dev=%u port=%d ep=%02x ret=%d active=%u paused_by=%x stalled=%u state=%u remaining=%u\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr, + ret, q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests); +} + +static void bce_vhci_transfer_queue_cancel_w(struct work_struct *work) +{ + unsigned long flags; + struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_cancel); + struct bce_vhci_urb *vurb; + struct urb *urb; + int status; + bool was_active; + + for (;;) { + spin_lock_irqsave(&q->urb_lock, flags); + if (list_empty(&q->cancel_urb_list)) { + spin_unlock_irqrestore(&q->urb_lock, flags); + return; + } + + vurb = list_first_entry(&q->cancel_urb_list, struct bce_vhci_urb, cancel_list); + list_del_init(&vurb->cancel_list); + urb = vurb->urb; + status = vurb->cancel_status; + was_active = vurb->cancel_was_active; + spin_unlock_irqrestore(&q->urb_lock, flags); + + /* Endpoint pause talks to bridgeOS and may sleep. */ + if (was_active) { + pr_debug("t2bce_vhci: [%02x] async cancelling URB\n", q->endp_addr); + bce_vhci_transfer_queue_pause(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + } + + spin_lock_irqsave(&q->urb_lock, flags); + if (was_active) + ++q->remaining_active_requests; + usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); + urb->hcpriv = NULL; + spin_unlock_irqrestore(&q->urb_lock, flags); + + usb_hcd_giveback_urb(q->vhci->hcd, urb, status); + + if (was_active) + bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ); + + bce_vhci_urb_free(vurb); + } +} + +void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q) +{ + queue_work(q->vhci->tq_state_wq, &q->w_reset); +} + +static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q) +{ + struct urb *urb, *urbt; + struct bce_vhci_urb *vurb; + list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) { + vurb = urb->hcpriv; + if (!bce_vhci_transfer_queue_can_init_urb(q)) + break; + if (vurb->state == BCE_VHCI_URB_INIT_PENDING) + bce_vhci_urb_init(vurb); + } +} + + + +static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout); + +static bool bce_vhci_urb_uses_sg(struct bce_vhci_urb *urb) +{ + return urb->sg_segments && urb->urb->sg && urb->urb->num_mapped_sgs > 0; +} + +static void bce_vhci_urb_sync_sg_segments_for_cpu(struct bce_vhci_urb *urb) +{ + struct device *dev; + + if (!urb->sg_segments_for_device) + return; + + dev = t2bce_core_client_dma_dev(urb->q->vhci->client); + dma_sync_single_for_cpu(dev, urb->sg_segments_dma, urb->sg_segments_size, + DMA_TO_DEVICE); + urb->sg_segments_for_device = false; +} + +static void bce_vhci_urb_free(struct bce_vhci_urb *urb) +{ + struct device *dev; + + if (!urb) + return; + + if (urb->sg_segments) { + bce_vhci_urb_sync_sg_segments_for_cpu(urb); + dev = t2bce_core_client_dma_dev(urb->q->vhci->client); + dma_unmap_single(dev, urb->sg_segments_dma, urb->sg_segments_size, + DMA_TO_DEVICE); + kfree(urb->sg_segments); + } + + kfree(urb); +} + +static int bce_vhci_urb_init_sg_segments(struct bce_vhci_urb *vurb) +{ + struct device *dev; + + if (!vurb->urb->sg || vurb->urb->num_mapped_sgs <= 0) + return 0; + + if (vurb->urb->num_mapped_sgs > SIZE_MAX / sizeof(*vurb->sg_segments)) + return -EOVERFLOW; + + vurb->sg_segment_count = vurb->urb->num_mapped_sgs; + vurb->sg_segments_size = vurb->sg_segment_count * sizeof(*vurb->sg_segments); + vurb->sg_segments = kcalloc(vurb->sg_segment_count, + sizeof(*vurb->sg_segments), GFP_KERNEL); + if (!vurb->sg_segments) + return -ENOMEM; + + dev = t2bce_core_client_dma_dev(vurb->q->vhci->client); + vurb->sg_segments_dma = dma_map_single(dev, vurb->sg_segments, + vurb->sg_segments_size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, vurb->sg_segments_dma)) { + kfree(vurb->sg_segments); + vurb->sg_segments = NULL; + vurb->sg_segments_size = 0; + vurb->sg_segment_count = 0; + return -ENOMEM; + } + vurb->sg_segments_for_device = true; + bce_vhci_urb_sync_sg_segments_for_cpu(vurb); + + return 0; +} + +static int bce_vhci_urb_set_sg_submission(struct bce_vhci_urb *urb, + struct t2bce_core_queue_sq *sq, u32 offset, u32 size) +{ + struct device *dev = t2bce_core_client_dma_dev(urb->q->vhci->client); + struct scatterlist *sg; + unsigned int i; + unsigned int segment_count = 0; + u32 skip = offset; + u32 remaining = size; + + if (!bce_vhci_urb_uses_sg(urb) || !size) + return -EINVAL; + + /* + * BCE consumes a compact address/length array, not Linux scatterlist + * metadata. Rebuild that array for the requested URB window. + */ + bce_vhci_urb_sync_sg_segments_for_cpu(urb); + + for_each_sg(urb->urb->sg, sg, urb->urb->num_mapped_sgs, i) { + dma_addr_t addr = sg_dma_address(sg); + u32 sg_len = sg_dma_len(sg); + u32 segment_len; + + if (!sg_len) + continue; + if (skip >= sg_len) { + skip -= sg_len; + continue; + } + + addr += skip; + sg_len -= skip; + skip = 0; + + segment_len = min(sg_len, remaining); + urb->sg_segments[segment_count].addr = addr; + urb->sg_segments[segment_count].length = segment_len; + segment_count++; + remaining -= segment_len; + + if (!remaining) + break; + } + + if (remaining || !segment_count) + return -EINVAL; + + dma_sync_single_for_device(dev, urb->sg_segments_dma, + segment_count * sizeof(*urb->sg_segments), DMA_TO_DEVICE); + urb->sg_segments_for_device = true; + t2bce_core_set_next_submission_segment_list(sq, urb->sg_segments_dma, + segment_count * sizeof(*urb->sg_segments)); + return 0; +} + +int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb) +{ + unsigned long flags; + int status = 0; + struct bce_vhci_urb *vurb; + bool is_control = (usb_endpoint_num(&urb->ep->desc) == 0); + + if (is_control && q->endp_addr == 0x00 && + !q->active && !q->stalled && + q->paused_by == BCE_VHCI_PAUSE_INTERNAL_WQ) { + pr_warn("t2bce_vhci: EP0 enqueue scheduling async stale internal pause recovery dev=%u port=%d state=%u remaining=%u urb_reject=%d\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), q->state, + q->remaining_active_requests, atomic_read(&urb->reject)); + queue_work(q->vhci->tq_state_wq, &q->w_resume); + } + + vurb = kzalloc(sizeof(struct bce_vhci_urb), GFP_KERNEL); + if (!vurb) + return -ENOMEM; + + urb->hcpriv = vurb; + + vurb->q = q; + vurb->urb = urb; + vurb->dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + vurb->is_control = is_control; + INIT_LIST_HEAD(&vurb->cancel_list); + status = bce_vhci_urb_init_sg_segments(vurb); + if (status) { + urb->hcpriv = NULL; + bce_vhci_urb_free(vurb); + return status; + } + + bce_vhci_urb_log_control(vurb, "enqueue"); + + spin_lock_irqsave(&q->urb_lock, flags); + if (vurb->is_control && (!q->active || q->stalled || q->paused_by)) + pr_debug("t2bce_vhci: EP0 enqueue dev=%u port=%d active=%u paused_by=%x stalled=%u state=%u remaining=%u urb_reject=%d\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests, atomic_read(&urb->reject)); + + status = usb_hcd_link_urb_to_ep(q->vhci->hcd, urb); + if (status) { + if (vurb->is_control) + pr_warn("t2bce_vhci: EP0 enqueue link failed dev=%u port=%d status=%d active=%u paused_by=%x stalled=%u state=%u remaining=%u urb_reject=%d\n", + q->dev_addr, bce_vhci_transfer_queue_port(q), status, + q->active, q->paused_by, q->stalled, q->state, + q->remaining_active_requests, atomic_read(&urb->reject)); + spin_unlock_irqrestore(&q->urb_lock, flags); + urb->hcpriv = NULL; + bce_vhci_urb_free(vurb); + return status; + } + + if (q->active) { + if (bce_vhci_transfer_queue_can_init_urb(vurb->q)) + status = bce_vhci_urb_init(vurb); + else + vurb->state = BCE_VHCI_URB_INIT_PENDING; + } else { + if (q->stalled) + bce_vhci_transfer_queue_request_reset(q); + vurb->state = BCE_VHCI_URB_INIT_PENDING; + } + if (status) { + usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb); + urb->hcpriv = NULL; + bce_vhci_urb_free(vurb); + } else { + bce_vhci_transfer_queue_deliver_pending(q); + } + spin_unlock_irqrestore(&q->urb_lock, flags); + pr_debug("t2bce_vhci: [%02x] URB enqueued (dir = %s, size = %i)\n", q->endp_addr, + usb_urb_dir_in(urb) ? "IN" : "OUT", urb->transfer_buffer_length); + return status; +} + +static int bce_vhci_urb_init(struct bce_vhci_urb *vurb) +{ + int status = 0; + + if (vurb->q->remaining_active_requests == 0) { + pr_err("t2bce_vhci: cannot init request (remaining_active_requests = 0)\n"); + return -EINVAL; + } + + if (vurb->is_control) { + vurb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST; + bce_vhci_urb_log_control(vurb, "init-wait-setup"); + } else { + status = bce_vhci_urb_data_start(vurb, NULL); + } + + if (!status) { + --vurb->q->remaining_active_requests; + } + return status; +} + +static void bce_vhci_urb_complete(struct bce_vhci_urb *urb, int status) +{ + struct bce_vhci_transfer_queue *q = urb->q; + struct bce_vhci *vhci = q->vhci; + struct urb *real_urb = urb->urb; + pr_debug("t2bce_vhci: [%02x] URB complete %i\n", q->endp_addr, status); + bce_vhci_urb_log_control(urb, "complete"); + usb_hcd_unlink_urb_from_ep(vhci->hcd, real_urb); + real_urb->hcpriv = NULL; + real_urb->status = status; + if (urb->state != BCE_VHCI_URB_INIT_PENDING) + ++urb->q->remaining_active_requests; + bce_vhci_urb_free(urb); + list_add_tail(&real_urb->urb_list, &q->giveback_urb_list); +} + +int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status) +{ + struct bce_vhci_urb *vurb; + unsigned long flags; + int ret; + enum bce_vhci_urb_state old_state; + + spin_lock_irqsave(&q->urb_lock, flags); + if ((ret = usb_hcd_check_unlink_urb(q->vhci->hcd, urb, status))) { + spin_unlock_irqrestore(&q->urb_lock, flags); + return ret; + } + + vurb = urb->hcpriv; + if (!vurb) { + spin_unlock_irqrestore(&q->urb_lock, flags); + return -ENOENT; + } + + old_state = vurb->state; + bce_vhci_urb_log_control(vurb, "cancel-request"); + + if (old_state == BCE_VHCI_URB_CANCELLED) { + spin_unlock_irqrestore(&q->urb_lock, flags); + pr_debug("t2bce_vhci: URB %p is already cancelled, skipping\n", urb); + return 0; + } + + vurb->state = BCE_VHCI_URB_CANCELLED; + vurb->cancel_status = status; + vurb->cancel_was_active = old_state != BCE_VHCI_URB_INIT_PENDING; + + if (list_empty(&vurb->cancel_list)) + list_add_tail(&vurb->cancel_list, &q->cancel_urb_list); + spin_unlock_irqrestore(&q->urb_lock, flags); + queue_work(q->vhci->tq_state_wq, &q->w_cancel); + + return 0; +} + +static int bce_vhci_urb_data_transfer_in(struct bce_vhci_urb *urb, unsigned long *timeout) +{ + struct bce_vhci_message msg; + u32 tr_len; + int reservation1, reservation2 = -EFAULT; + bool uses_sg; + int status; + + pr_debug("t2bce_vhci: [%02x] DMA from device %llx %x\n", urb->q->endp_addr, + (u64) urb->urb->transfer_dma, urb->urb->transfer_buffer_length); + + /* The firmware request and the inbound DMA submission must be queued as a pair. */ + reservation1 = t2bce_core_reserve_submission(urb->q->vhci->msg_asynchronous.sq, timeout); + if (!reservation1) + reservation2 = t2bce_core_reserve_submission(urb->q->sq_in, timeout); + if (reservation1 || reservation2) { + pr_err("t2bce_vhci: Failed to reserve a submission for URB data transfer\n"); + if (!reservation1) + t2bce_core_cancel_submission_reservation(urb->q->vhci->msg_asynchronous.sq); + return -ENOMEM; + } + + urb->send_offset = urb->receive_offset; + + tr_len = urb->urb->transfer_buffer_length - urb->send_offset; + uses_sg = bce_vhci_urb_uses_sg(urb); + + if (uses_sg) { + status = bce_vhci_urb_set_sg_submission(urb, urb->q->sq_in, + urb->send_offset, tr_len); + if (status) { + t2bce_core_cancel_submission_reservation(urb->q->sq_in); + t2bce_core_cancel_submission_reservation(urb->q->vhci->msg_asynchronous.sq); + pr_err("t2bce_vhci: [%02x] failed to prepare SG IN submission: %d\n", + urb->q->endp_addr, status); + return status; + } + } + + spin_lock(&urb->q->vhci->msg_asynchronous_lock); + msg.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST; + msg.status = 0; + msg.param1 = ((urb->urb->ep->desc.bEndpointAddress & 0x8Fu) << 8) | urb->q->dev_addr; + msg.param2 = tr_len; + bce_vhci_message_queue_write(&urb->q->vhci->msg_asynchronous, &msg); + spin_unlock(&urb->q->vhci->msg_asynchronous_lock); + + if (!uses_sg) + t2bce_core_set_next_submission_single(urb->q->sq_in, + urb->urb->transfer_dma + urb->send_offset, tr_len); + t2bce_core_submit_to_device(urb->q->sq_in); + + urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; + return 0; +} + +static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout) +{ + if (urb->dir == DMA_TO_DEVICE) { + if (urb->urb->transfer_buffer_length > 0) + urb->state = BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST; + else + urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; + bce_vhci_urb_log_control(urb, "data-start-out"); + return 0; + } else { + bce_vhci_urb_log_control(urb, "data-start-in"); + return bce_vhci_urb_data_transfer_in(urb, timeout); + } +} + +static int bce_vhci_urb_send_out_data(struct bce_vhci_urb *urb, dma_addr_t addr, size_t size) +{ + unsigned long timeout = 0; + if (t2bce_core_reserve_submission(urb->q->sq_out, &timeout)) { + pr_err("t2bce_vhci: Failed to reserve a submission for URB data transfer\n"); + return -EPIPE; + } + + pr_debug("t2bce_vhci: [%02x] DMA to device %llx %lx\n", urb->q->endp_addr, (u64) addr, size); + + t2bce_core_set_next_submission_single(urb->q->sq_out, addr, size); + atomic_inc(&urb->q->sq_out_pending); + t2bce_core_submit_to_device(urb->q->sq_out); + return 0; +} + +static int bce_vhci_urb_send_out_transfer_data(struct bce_vhci_urb *urb, + u32 offset, size_t size) +{ + unsigned long timeout = 0; + int status; + + if (t2bce_core_reserve_submission(urb->q->sq_out, &timeout)) { + pr_err("t2bce_vhci: Failed to reserve a submission for URB data transfer\n"); + return -EPIPE; + } + + if (bce_vhci_urb_uses_sg(urb)) { + status = bce_vhci_urb_set_sg_submission(urb, urb->q->sq_out, + offset, size); + if (status) { + t2bce_core_cancel_submission_reservation(urb->q->sq_out); + pr_err("t2bce_vhci: [%02x] failed to prepare SG OUT submission: %d\n", + urb->q->endp_addr, status); + return status; + } + } else { + t2bce_core_set_next_submission_single(urb->q->sq_out, + urb->urb->transfer_dma + offset, size); + } + + atomic_inc(&urb->q->sq_out_pending); + t2bce_core_submit_to_device(urb->q->sq_out); + return 0; +} + +static int bce_vhci_urb_data_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + u32 tr_len; + int status; + if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST) { + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { + tr_len = min(urb->urb->transfer_buffer_length - urb->send_offset, (u32) msg->param2); + bce_vhci_urb_log_control(urb, "data-request"); + if ((status = bce_vhci_urb_send_out_transfer_data(urb, urb->send_offset, tr_len))) + return status; + urb->send_offset += tr_len; + urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION; + bce_vhci_urb_log_control(urb, "data-sent"); + return 0; + } + } + + /* OUT endpoints can receive extra transfer requests while waiting for host data. */ + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) + return -EAGAIN; + pr_err("t2bce_vhci: [%02x] %s URB unexpected message (state = %x, msg: %x %x %x %llx)\n", + urb->q->endp_addr, (urb->is_control ? "Control (data update)" : "Data"), urb->state, + msg->cmd, msg->status, msg->param1, msg->param2); + return -EAGAIN; +} + +static int bce_vhci_urb_data_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c) +{ + bce_vhci_urb_sync_sg_segments_for_cpu(urb); + + if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + urb->receive_offset += c->data_size; + if (urb->dir == DMA_FROM_DEVICE || urb->receive_offset >= urb->urb->transfer_buffer_length) { + urb->urb->actual_length = (u32) urb->receive_offset; + urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE; + bce_vhci_urb_log_control(urb, "data-complete"); + if (!urb->is_control) { + bce_vhci_urb_complete(urb, 0); + return -ENOENT; + } + } + } else { + pr_warn_ratelimited("t2bce_vhci: [%02x] data URB unexpected completion state=%x status=%x size=%llu\n", + urb->q->endp_addr, urb->state, c->status, c->data_size); + } + return 0; +} + + +static int bce_vhci_urb_control_check_status(struct bce_vhci_urb *urb) +{ + struct bce_vhci_transfer_queue *q = urb->q; + if (urb->received_status == 0) + return 0; + if (urb->state == BCE_VHCI_URB_DATA_TRANSFER_COMPLETE || + (urb->received_status != BCE_VHCI_SUCCESS && urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST && + urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION)) { + urb->state = BCE_VHCI_URB_CONTROL_COMPLETE; + if (urb->received_status != BCE_VHCI_SUCCESS) { + if (urb->received_status == 3 && urb->q->endp_addr == 0x00) { + q->active = false; + q->stalled = true; + q->state = BCE_VHCI_ENDPOINT_STALLED; + pr_debug("t2bce_vhci: EP0 status=3 dev=%u state=%x; completing urb and scheduling endpoint reset paused_by=%x remaining=%u\n", + q->dev_addr, urb->state, q->paused_by, + q->remaining_active_requests); + bce_vhci_urb_complete(urb, -EPIPE); + bce_vhci_transfer_queue_request_reset(q); + return -ENOENT; + } + + pr_err("t2bce_vhci: [%02x] URB failed: %x\n", + urb->q->endp_addr, urb->received_status); + urb->q->active = false; + urb->q->stalled = true; + bce_vhci_urb_complete(urb, -EPIPE); + if (!list_empty(&q->endp->urb_list)) + bce_vhci_transfer_queue_request_reset(q); + return -ENOENT; + } + bce_vhci_urb_complete(urb, 0); + return -ENOENT; + } + return 0; +} + +static int bce_vhci_urb_control_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + int status; + if (msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { + urb->received_status = msg->status; + bce_vhci_urb_log_control(urb, "status-msg"); + return bce_vhci_urb_control_check_status(urb); + } + + if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST) { + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) { + bce_vhci_urb_log_control(urb, "setup-request"); + if (bce_vhci_urb_send_out_data(urb, urb->urb->setup_dma, sizeof(struct usb_ctrlrequest))) { + pr_err("t2bce_vhci: [%02x] Failed to start URB setup transfer\n", urb->q->endp_addr); + /* Keep the old state machine behavior; endpoint recovery is driven by later firmware status. */ + return 0; + } + urb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION; + bce_vhci_urb_log_control(urb, "setup-sent"); + pr_debug("t2bce_vhci: [%02x] Sent setup %llx\n", urb->q->endp_addr, urb->urb->setup_dma); + return 0; + } + } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || + urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + if ((status = bce_vhci_urb_data_update(urb, msg))) + return status; + return bce_vhci_urb_control_check_status(urb); + } + + /* OUT endpoints can receive extra transfer requests while waiting for host data. */ + if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL) + return -EAGAIN; + pr_err("t2bce_vhci: [%02x] Control URB unexpected message (state = %x, msg: %x %x %x %llx)\n", urb->q->endp_addr, + urb->state, msg->cmd, msg->status, msg->param1, msg->param2); + return -EAGAIN; +} + +static int bce_vhci_urb_control_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c) +{ + int status; + unsigned long timeout; + + if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION) { + if (c->data_size != sizeof(struct usb_ctrlrequest)) + pr_err("t2bce_vhci: [%02x] transfer complete data size mistmatch for usb_ctrlrequest (%llx instead of %lx)\n", + urb->q->endp_addr, c->data_size, sizeof(struct usb_ctrlrequest)); + + bce_vhci_urb_log_control(urb, "setup-complete"); + timeout = 1000; + status = bce_vhci_urb_data_start(urb, &timeout); + if (status) { + bce_vhci_urb_complete(urb, status); + return -ENOENT; + } + return 0; + } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST || + urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + if ((status = bce_vhci_urb_data_transfer_completion(urb, c))) + return status; + return bce_vhci_urb_control_check_status(urb); + } else { + pr_err("t2bce_vhci: [%02x] Control URB unexpected completion (state = %x)\n", urb->q->endp_addr, urb->state); + } + return 0; +} + +static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg) +{ + if (urb->state == BCE_VHCI_URB_INIT_PENDING) + return -EAGAIN; + if (urb->is_control) + return bce_vhci_urb_control_update(urb, msg); + else + return bce_vhci_urb_data_update(urb, msg); +} + +static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c) +{ + if (urb->is_control) + return bce_vhci_urb_control_transfer_completion(urb, c); + else + return bce_vhci_urb_data_transfer_completion(urb, c); +} + +static void bce_vhci_urb_resume(struct bce_vhci_urb *urb) +{ + int status = 0; + if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) { + status = bce_vhci_urb_data_transfer_in(urb, NULL); + } + if (status) + bce_vhci_urb_complete(urb, status); +} diff --git a/drivers/staging/t2bce/t2bce_vhci/transfer.h b/drivers/staging/t2bce/t2bce_vhci/transfer.h new file mode 100644 index 000000000..5416b65c5 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_vhci/transfer.h @@ -0,0 +1,86 @@ +#ifndef BCEDRIVER_TRANSFER_H +#define BCEDRIVER_TRANSFER_H + +#include +#include +#include "queue.h" +#include "command.h" + +struct bce_vhci_list_message { + struct list_head list; + struct bce_vhci_message msg; +}; +enum bce_vhci_pause_source { + BCE_VHCI_PAUSE_INTERNAL_WQ = 1, + BCE_VHCI_PAUSE_FIRMWARE = 2, + BCE_VHCI_PAUSE_SUSPEND = 4, + BCE_VHCI_PAUSE_SHUTDOWN = 8 +}; +struct bce_vhci_transfer_queue { + struct bce_vhci *vhci; + struct usb_host_endpoint *endp; + enum bce_vhci_endpoint_state state; + u32 max_active_requests, remaining_active_requests; + bool active, stalled; + u32 paused_by; + bce_vhci_device_t dev_addr; + u8 endp_addr; + struct t2bce_core_queue_cq *cq; + struct t2bce_core_queue_sq *sq_in; + struct t2bce_core_queue_sq *sq_out; + struct list_head evq; + struct spinlock urb_lock; + struct mutex pause_lock; + struct list_head giveback_urb_list; + wait_queue_head_t sq_out_wait_queue; + atomic_t sq_out_pending; + struct list_head cancel_urb_list; + + struct work_struct w_reset; + struct work_struct w_resume; + struct work_struct w_cancel; +}; +enum bce_vhci_urb_state { + BCE_VHCI_URB_INIT_PENDING, + BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST, + BCE_VHCI_URB_WAITING_FOR_COMPLETION, + BCE_VHCI_URB_DATA_TRANSFER_COMPLETE, + BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST, + BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION, + BCE_VHCI_URB_CONTROL_COMPLETE, + BCE_VHCI_URB_CANCELLED +}; +struct bce_vhci_urb { + struct urb *urb; + struct bce_vhci_transfer_queue *q; + enum dma_data_direction dir; + bool is_control; + enum bce_vhci_urb_state state; + int received_status; + u32 send_offset; + u32 receive_offset; + struct t2bce_core_dma_segment *sg_segments; + dma_addr_t sg_segments_dma; + size_t sg_segments_size; + unsigned int sg_segment_count; + bool sg_segments_for_device; + struct list_head cancel_list; + int cancel_status; + bool cancel_was_active; +}; + +void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q, + struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir); +void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q); +void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg); +int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q); +int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q); +int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); +int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src); +int bce_vhci_transfer_queue_suspend_pause(struct bce_vhci_transfer_queue *q); +void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q); + +int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb); +int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status); + +#endif //BCEDRIVER_TRANSFER_H diff --git a/drivers/staging/t2bce/t2bce_vhci/vhci.c b/drivers/staging/t2bce/t2bce_vhci/vhci.c new file mode 100644 index 000000000..682badb23 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_vhci/vhci.c @@ -0,0 +1,1293 @@ +#include "vhci.h" +#include "command.h" +#include +#include +#include +#include +#include +#include +#include + +/* + * bridgeOS exposes several T2-internal devices through a virtual USB 2.0 hub. + * This HCD translates usbcore hub, device and endpoint operations into the + * firmware command/event protocol carried over BCE queues. + */ + +static dev_t bce_vhci_chrdev; +static struct class *bce_vhci_class; +static const struct hc_driver bce_vhci_driver; +static u16 bce_vhci_port_mask = U16_MAX; +static struct bce_vhci *global_vhci; + +enum bce_vhci_port_status_bits { + BCE_VHCI_PORT_STATUS_OVERCURRENT = 0x2, + BCE_VHCI_PORT_STATUS_CONNECTED = 0x4, + BCE_VHCI_PORT_STATUS_RESET = 0x8, + BCE_VHCI_PORT_STATUS_ENABLED = 0x10, + BCE_VHCI_PORT_STATUS_SUSPENDED_LEGACY = 0x60, + BCE_VHCI_PORT_STATUS_SUSPENDED = 0x80, + BCE_VHCI_PORT_STATUS_HIGH_SPEED = 0x200, + BCE_VHCI_PORT_STATUS_C_CONNECTION = 0x40000, +}; + +static int bce_vhci_create_event_queues(struct bce_vhci *vhci); +static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci); +static int bce_vhci_create_message_queues(struct bce_vhci *vhci); +static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci); +static void bce_vhci_handle_firmware_events_w(struct work_struct *ws); +static void bce_vhci_add_hcd_w(struct work_struct *ws); +static void bce_vhci_firmware_event_completion(struct t2bce_core_queue_sq *sq); +static int bce_vhci_start_controller(struct bce_vhci *vhci); +static void bce_vhci_forget_devices(struct bce_vhci *vhci); +static int __bce_vhci_add_hcd(struct bce_vhci *vhci); +static void bce_vhci_shutdown_client(void *userdata); +static void bce_vhci_pm_reset_client(void *userdata); +static void bce_vhci_pm_prepare_client(void *userdata); +static void bce_vhci_pm_prepare_no_state_client(void *userdata); +static void bce_vhci_pm_mark_no_state_resume_client(void *userdata); +static bool bce_vhci_pm_is_no_state_resume_client(void *userdata); +static void bce_vhci_pm_complete_client(void *userdata); + +static const struct t2bce_core_client_pm_ops bce_vhci_pm_ops = { + .shutdown = bce_vhci_shutdown_client, + .pm_reset = bce_vhci_pm_reset_client, + .pm_prepare = bce_vhci_pm_prepare_client, + .pm_prepare_no_state = bce_vhci_pm_prepare_no_state_client, + .pm_mark_no_state_resume = bce_vhci_pm_mark_no_state_resume_client, + .pm_is_no_state_resume = bce_vhci_pm_is_no_state_resume_client, + .pm_complete = bce_vhci_pm_complete_client, +}; + +int bce_vhci_create(struct device *parent, struct bce_vhci *vhci) +{ + int status; + + spin_lock_init(&vhci->hcd_spinlock); + + vhci->vdevt = bce_vhci_chrdev; + vhci->vdev = device_create(bce_vhci_class, parent, vhci->vdevt, NULL, "t2bce_vhci"); + if (IS_ERR_OR_NULL(vhci->vdev)) { + status = PTR_ERR(vhci->vdev); + goto fail_dev; + } + vhci->client = t2bce_core_client_get(vhci->vdev); + if (IS_ERR(vhci->client)) { + status = PTR_ERR(vhci->client); + vhci->client = NULL; + goto fail_client; + } + t2bce_core_client_set_pm_ops(vhci->client, &bce_vhci_pm_ops, vhci); + + if ((status = bce_vhci_create_message_queues(vhci))) + goto fail_mq; + if ((status = bce_vhci_create_event_queues(vhci))) + goto fail_eq; + + vhci->tq_state_wq = alloc_ordered_workqueue("t2bce_vhci-tq-state", 0); + INIT_WORK(&vhci->w_fw_events, bce_vhci_handle_firmware_events_w); + INIT_WORK(&vhci->w_add_hcd, bce_vhci_add_hcd_w); + vhci->port_change_pending = 0; + vhci->stateful_suspended_bounce = 0; + vhci->stateful_resume = false; + + vhci->hcd = usb_create_hcd(&bce_vhci_driver, vhci->vdev, "t2bce_vhci"); + if (!vhci->hcd) { + status = -ENOMEM; + goto fail_hcd; + } + vhci->hcd->self.sysdev = t2bce_core_client_dma_dev(vhci->client); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) + vhci->hcd->self.uses_dma = 1; +#endif + *((struct bce_vhci **) vhci->hcd->hcd_priv) = vhci; + vhci->hcd->speed = HCD_USB2; + + if ((status = __bce_vhci_add_hcd(vhci))) + goto fail_hcd; + + pr_info("t2bce_vhci: initialized\n"); + return 0; + +fail_hcd: + bce_vhci_destroy_event_queues(vhci); +fail_eq: + bce_vhci_destroy_message_queues(vhci); +fail_mq: + t2bce_core_client_set_pm_ops(vhci->client, NULL, NULL); + t2bce_core_client_put(vhci->client); + vhci->client = NULL; +fail_client: + device_destroy(bce_vhci_class, vhci->vdevt); +fail_dev: + if (!status) + status = -EINVAL; + return status; +} + +void bce_vhci_destroy(struct bce_vhci *vhci) +{ + cancel_work_sync(&vhci->w_add_hcd); + bce_vhci_remove_hcd(vhci); + bce_vhci_destroy_event_queues(vhci); + bce_vhci_destroy_message_queues(vhci); + t2bce_core_client_set_pm_ops(vhci->client, NULL, NULL); + t2bce_core_client_put(vhci->client); + vhci->client = NULL; + device_destroy(bce_vhci_class, vhci->vdevt); +} + +static int __bce_vhci_add_hcd(struct bce_vhci *vhci) +{ + int status; + + status = usb_add_hcd(vhci->hcd, 0, 0); + if (!status) + vhci->hcd_registered = true; + return status; +} + +int bce_vhci_add_hcd(struct bce_vhci *vhci) +{ + if (vhci->hcd_registered) + return 0; + return __bce_vhci_add_hcd(vhci); +} + +void bce_vhci_remove_hcd(struct bce_vhci *vhci) +{ + cancel_work_sync(&vhci->w_add_hcd); + if (!vhci->hcd_registered) + return; + usb_remove_hcd(vhci->hcd); + vhci->hcd_registered = false; +} + +void bce_vhci_shutdown(struct bce_vhci *vhci) +{ + cancel_work_sync(&vhci->w_add_hcd); + if (vhci->hcd_registered) { + bce_vhci_remove_hcd(vhci); + return; + } + + /* + * Shutdown is not remove: the USB core may already have detached the HCD, + * but the T2-side controller can still be left enabled. Send the controller + * disable directly as a final lights-out command while queues still exist. + */ + bce_vhci_cmd_controller_disable(&vhci->cq); +} + +void bce_vhci_pm_reset(struct bce_vhci *vhci) +{ + vhci->no_state_resume = false; +} + +void bce_vhci_pm_prepare(struct bce_vhci *vhci) +{ + cancel_work_sync(&vhci->w_add_hcd); + vhci->port_change_pending = 0; +} + +void bce_vhci_pm_prepare_no_state(struct bce_vhci *vhci) +{ + bce_vhci_remove_hcd(vhci); +} + +void bce_vhci_pm_mark_no_state_resume(struct bce_vhci *vhci) +{ + vhci->no_state_resume = true; +} + +bool bce_vhci_pm_is_no_state_resume(struct bce_vhci *vhci) +{ + return vhci->no_state_resume; +} + +void bce_vhci_pm_complete(struct bce_vhci *vhci) +{ + if (!vhci->no_state_resume) + return; + + /* Re-add the VHCI HCD after the PM core completed resume ordering. */ + pr_debug("t2bce_vhci: scheduling HCD re-add after no-state wake\n"); + queue_work(vhci->tq_state_wq, &vhci->w_add_hcd); +} + +static void bce_vhci_pm_reset_client(void *userdata) +{ + bce_vhci_pm_reset(userdata); +} + +static void bce_vhci_pm_prepare_client(void *userdata) +{ + bce_vhci_pm_prepare(userdata); +} + +static void bce_vhci_shutdown_client(void *userdata) +{ + bce_vhci_shutdown(userdata); +} + +static void bce_vhci_pm_prepare_no_state_client(void *userdata) +{ + bce_vhci_pm_prepare_no_state(userdata); +} + +static void bce_vhci_pm_mark_no_state_resume_client(void *userdata) +{ + bce_vhci_pm_mark_no_state_resume(userdata); +} + +static bool bce_vhci_pm_is_no_state_resume_client(void *userdata) +{ + return bce_vhci_pm_is_no_state_resume(userdata); +} + +static void bce_vhci_pm_complete_client(void *userdata) +{ + bce_vhci_pm_complete(userdata); +} + +static void bce_vhci_add_hcd_w(struct work_struct *ws) +{ + struct bce_vhci *vhci = container_of(ws, struct bce_vhci, w_add_hcd); + int status; + + pr_debug("t2bce_vhci: deferred HCD add after no-state wake\n"); + status = bce_vhci_add_hcd(vhci); + if (status) { + pr_err("t2bce_vhci: deferred HCD add failed: %d\n", status); + return; + } + + vhci->no_state_resume = false; +} + +struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd) +{ + return *((struct bce_vhci **) hcd->hcd_priv); +} + +int bce_vhci_start(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + return bce_vhci_start_controller(vhci); +} + +void bce_vhci_stop(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_cmd_controller_disable(&vhci->cq); +} + +static int bce_vhci_start_controller(struct bce_vhci *vhci) +{ + int status; + u16 port_mask = 0; + bce_vhci_port_t port_no = 0; + + if ((status = bce_vhci_cmd_controller_enable(&vhci->cq, 1, &port_mask))) + return status; + vhci->port_mask = port_mask; + vhci->port_power_mask = 0; + if ((status = bce_vhci_cmd_controller_start(&vhci->cq))) + return status; + while (port_mask) { + port_no += 1; + port_mask >>= 1; + } + vhci->port_count = port_no; + return 0; +} + +static void bce_vhci_forget_devices(struct bce_vhci *vhci) +{ + int i, j; + bce_vhci_device_t devid; + struct bce_vhci_device *dev; + + for (i = 0; i < 16; i++) { + devid = vhci->port_to_device[i]; + if (!devid) + continue; + + dev = vhci->devices[devid]; + if (!dev) { + vhci->port_to_device[i] = 0; + continue; + } + + for (j = 0; j < 32; j++) { + if (!(dev->tq_mask & BIT(j))) + continue; + if (dev->tq[j].endp) + dev->tq[j].endp->hcpriv = NULL; + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[j]); + } + + dev->tq_mask = 0; + vhci->devices[devid] = NULL; + vhci->port_to_device[i] = 0; + kfree(dev); + } +} + +static int bce_vhci_hub_status_data(struct usb_hcd *hcd, char *buf) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + unsigned long pending; + + /* Only ports reported by firmware as changed should wake the hub state machine. */ + pending = READ_ONCE(vhci->port_change_pending); + if (!pending) + return 0; + + memset(buf, 0, 2); + buf[0] = (char)((pending << 1) & 0xff); + buf[1] = (char)((pending >> 7) & 0xff); + return buf[0] || buf[1]; +} + +static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout); + +static int bce_vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex, char *buf, u16 wLength) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + struct usb_hub_descriptor *hd; + struct usb_hub_status *hs; + struct usb_port_status *ps; + u32 port_status; + u32 raw_port_status; + if (typeReq == GetHubDescriptor && wLength >= sizeof(struct usb_hub_descriptor)) { + hd = (struct usb_hub_descriptor *) buf; + memset(hd, 0, sizeof(*hd)); + hd->bDescLength = sizeof(struct usb_hub_descriptor); + hd->bDescriptorType = USB_DT_HUB; + hd->bNbrPorts = (u8) vhci->port_count; + hd->wHubCharacteristics = HUB_CHAR_INDV_PORT_LPSM | HUB_CHAR_INDV_PORT_OCPM; + hd->bPwrOn2PwrGood = 0; + hd->bHubContrCurrent = 0; + return 0; + } else if (typeReq == GetHubStatus && wLength >= sizeof(struct usb_hub_status)) { + hs = (struct usb_hub_status *) buf; + memset(hs, 0, sizeof(*hs)); + hs->wHubStatus = 0; + hs->wHubChange = 0; + return 0; + } else if (typeReq == GetPortStatus && wLength >= 4 /* usb 2.0 */) { + ps = (struct usb_port_status *) buf; + ps->wPortStatus = 0; + ps->wPortChange = 0; + + if (vhci->port_power_mask & BIT(wIndex)) + ps->wPortStatus |= USB_PORT_STAT_POWER; + + if (!(bce_vhci_port_mask & BIT(wIndex))) + return 0; + + if ((status = bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0, &port_status))) + return status; + raw_port_status = port_status; + + /* + * During stateful resume the T2 can briefly report a disconnect bounce + * for an already known virtual port. Keep usbcore from treating that + * transient as a real unplug; the follow-up status/event decides the + * actual port state. + */ + if (vhci->stateful_resume && + (port_status & BCE_VHCI_PORT_STATUS_C_CONNECTION) && + !(port_status & BCE_VHCI_PORT_STATUS_CONNECTED) && + wIndex < ARRAY_SIZE(vhci->port_to_device) && + vhci->port_to_device[wIndex]) { + port_status |= BCE_VHCI_PORT_STATUS_CONNECTED | + BCE_VHCI_PORT_STATUS_ENABLED | + BCE_VHCI_PORT_STATUS_SUSPENDED; + pr_debug("t2bce_vhci: hub masked stateful disconnect bounce port=%u raw=%x masked_raw=%x\n", + wIndex, raw_port_status, port_status); + } + if (port_status & BCE_VHCI_PORT_STATUS_ENABLED) + ps->wPortStatus |= USB_PORT_STAT_ENABLE; + if (port_status & BCE_VHCI_PORT_STATUS_CONNECTED) + ps->wPortStatus |= USB_PORT_STAT_CONNECTION; + if (port_status & BCE_VHCI_PORT_STATUS_OVERCURRENT) + ps->wPortStatus |= USB_PORT_STAT_OVERCURRENT; + if (port_status & BCE_VHCI_PORT_STATUS_RESET) + ps->wPortStatus |= USB_PORT_STAT_RESET; + if (port_status & BCE_VHCI_PORT_STATUS_HIGH_SPEED) + ps->wPortStatus |= USB_PORT_STAT_HIGH_SPEED; + if (port_status & (BCE_VHCI_PORT_STATUS_SUSPENDED | + BCE_VHCI_PORT_STATUS_SUSPENDED_LEGACY)) { + ps->wPortStatus |= USB_PORT_STAT_SUSPEND; + if (vhci->stateful_resume && (port_status & BCE_VHCI_PORT_STATUS_CONNECTED)) + ps->wPortStatus |= USB_PORT_STAT_ENABLE; + } + + /* + * These ports are fixed T2-internal devices, not user-pluggable ports. + * For stateful resume, suppress firmware connection-change bounces so + * usbcore does not re-enumerate solely because the T2 toggled its + * internal representation while restoring state. + */ + if (vhci->stateful_resume && (port_status & BCE_VHCI_PORT_STATUS_C_CONNECTION)) { + u32 cleared_status = 0; + + clear_bit(wIndex - 1, &vhci->port_change_pending); + status = bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0x40000, &cleared_status); + pr_debug("t2bce_vhci: hub suppressed C_CONNECTION port=%u raw=%x clear_status=%d cleared_raw=%x\n", + wIndex, port_status, status, cleared_status); + } else if (port_status & BCE_VHCI_PORT_STATUS_C_CONNECTION) { + ps->wPortChange |= USB_PORT_STAT_C_CONNECTION; + } + pr_debug("t2bce_vhci: hub GetPortStatus port=%u raw=%x status=%x change=%x\n", + wIndex, port_status, ps->wPortStatus, ps->wPortChange); + return 0; + } else if (typeReq == SetPortFeature) { + if (wValue == USB_PORT_FEAT_POWER) { + status = bce_vhci_cmd_port_power_on(&vhci->cq, (u8) wIndex); + /* usbcore tracks logical power state separately from bridgeOS. */ + if (!status) + vhci->port_power_mask |= BIT(wIndex); + return status; + } + if (wValue == USB_PORT_FEAT_RESET) { + pr_debug("t2bce_vhci: hub SetPortFeature RESET port=%u\n", wIndex); + return bce_vhci_reset_device(vhci, wIndex, wValue); + } + if (wValue == USB_PORT_FEAT_SUSPEND) { + /* Trace the usbcore hub state machine against the firmware port commands. */ + pr_debug("t2bce_vhci: hub SetPortFeature SUSPEND port=%u\n", wIndex); + if (vhci->system_suspending) { + pr_debug("t2bce_vhci: hub SetPortFeature SUSPEND port=%u skipped during system suspend\n", + wIndex); + return 0; + } + status = bce_vhci_cmd_port_suspend(&vhci->cq, (u8) wIndex); + pr_debug("t2bce_vhci: hub SetPortFeature SUSPEND port=%u -> %d\n", wIndex, status); + return status; + } + } else if (typeReq == ClearPortFeature) { + if (wValue == USB_PORT_FEAT_ENABLE) + return bce_vhci_cmd_port_disable(&vhci->cq, (u8) wIndex); + if (wValue == USB_PORT_FEAT_POWER) { + status = bce_vhci_cmd_port_power_off(&vhci->cq, (u8) wIndex); + if (!status) + vhci->port_power_mask &= ~BIT(wIndex); + return status; + } + if (wValue == USB_PORT_FEAT_C_CONNECTION) { + pr_debug("t2bce_vhci: hub ClearPortFeature C_CONNECTION port=%u\n", wIndex); + clear_bit(wIndex - 1, &vhci->port_change_pending); + return bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0x40000, &port_status); + } + if (wValue == USB_PORT_FEAT_C_RESET) { + pr_debug("t2bce_vhci: hub ClearPortFeature C_RESET port=%u\n", wIndex); + return 0; + } + if (wValue == USB_PORT_FEAT_SUSPEND) { + pr_debug("t2bce_vhci: hub ClearPortFeature SUSPEND port=%u\n", wIndex); + status = bce_vhci_cmd_port_resume(&vhci->cq, (u8) wIndex); + pr_debug("t2bce_vhci: hub ClearPortFeature SUSPEND port=%u -> %d\n", wIndex, status); + return status; + } + } + pr_err("t2bce_vhci: bce_vhci_hub_control unhandled request: %x %i %i [bufl=%i]\n", typeReq, wValue, wIndex, wLength); + dump_stack(); + return -EIO; +} + +static int bce_vhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + struct bce_vhci_device *vdev; + bce_vhci_device_t devid; + pr_debug("t2bce_vhci: enable_device port=%u\n", udev->portnum); + + if (vhci->port_to_device[udev->portnum]) + return 0; + + /* bridgeOS requires a firmware device id before endpoints are created. */ + if (bce_vhci_cmd_device_create(&vhci->cq, udev->portnum, &devid)) + return -EIO; + + pr_debug("t2bce_vhci: device_create port=%u dev=%u\n", udev->portnum, devid); + + vdev = kzalloc(sizeof(struct bce_vhci_device), GFP_KERNEL); + vhci->port_to_device[udev->portnum] = devid; + vhci->devices[devid] = vdev; + + bce_vhci_create_transfer_queue(vhci, &vdev->tq[0], &udev->ep0, devid, DMA_BIDIRECTIONAL); + udev->ep0.hcpriv = &vdev->tq[0]; + vdev->tq_mask |= BIT(0); + + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &udev->ep0.desc); + return 0; +} + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,8,0) +static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev) +#else +static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev, unsigned int timeout_ms) +#endif +{ + /* + * Older kernels create the firmware device from enable_device(); newer + * kernels use address_device() for the same bridgeOS operation. The + * firmware command path does not currently consume the usbcore timeout. + */ + return bce_vhci_enable_device(hcd, udev); +} + +static void bce_vhci_free_device(struct usb_hcd *hcd, struct usb_device *udev) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int i; + bce_vhci_device_t devid; + struct bce_vhci_device *dev; + pr_debug("t2bce_vhci: free_device port=%u\n", udev->portnum); + if (!vhci->port_to_device[udev->portnum]) + return; + devid = vhci->port_to_device[udev->portnum]; + dev = vhci->devices[devid]; + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, dev->tq[i].endp_addr); + if (dev->tq[i].endp) + dev->tq[i].endp->hcpriv = NULL; + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); + } + } + vhci->devices[devid] = NULL; + vhci->port_to_device[udev->portnum] = 0; + bce_vhci_cmd_device_destroy(&vhci->cq, devid); + kfree(dev); +} + +static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout) +{ + struct bce_vhci_device *dev = NULL; + bce_vhci_device_t devid; + int i; + int status; + enum dma_data_direction dir; + pr_debug("t2bce_vhci: reset_device port=%d\n", index); + + devid = vhci->port_to_device[index]; + if (devid) { + dev = vhci->devices[devid]; + + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN); + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, dev->tq[i].endp_addr); + if (dev->tq[i].endp) + dev->tq[i].endp->hcpriv = NULL; + bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]); + } + } + vhci->devices[devid] = NULL; + vhci->port_to_device[index] = 0; + bce_vhci_cmd_device_destroy(&vhci->cq, devid); + } + status = bce_vhci_cmd_port_reset(&vhci->cq, (u8) index, timeout); + + if (dev) { + if ((status = bce_vhci_cmd_device_create(&vhci->cq, index, &devid))) + return status; + vhci->devices[devid] = dev; + vhci->port_to_device[index] = devid; + + for (i = 0; i < 32; i++) { + if (dev->tq_mask & BIT(i)) { + dir = usb_endpoint_dir_in(&dev->tq[i].endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + if (i == 0) + dir = DMA_BIDIRECTIONAL; + bce_vhci_create_transfer_queue(vhci, &dev->tq[i], dev->tq[i].endp, devid, dir); + dev->tq[i].endp->hcpriv = &dev->tq[i]; + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &dev->tq[i].endp->desc); + } + } + } + + return status; +} + +static int bce_vhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev) +{ + return 0; +} + +static int bce_vhci_get_frame_number(struct usb_hcd *hcd) +{ + return 0; +} + +static int bce_vhci_suspend_quiesce(struct usb_hcd *hcd) +{ + int i, j; + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + struct bce_vhci_transfer_queue *tq; + struct urb *urb; + struct bce_vhci_urb *vurb; + unsigned long flags; + + pr_debug("t2bce_vhci: suspend started\n"); + + pr_debug("t2bce_vhci: suspend endpoints\n"); + for (i = 0; i < 16; i++) { + if (!vhci->port_to_device[i]) + continue; + for (j = 0; j < 32; j++) { + if (!(vhci->devices[vhci->port_to_device[i]]->tq_mask & BIT(j))) + continue; + tq = &vhci->devices[vhci->port_to_device[i]]->tq[j]; + if (tq->endp_addr == 0x00 || (usb_endpoint_xfer_int(&tq->endp->desc) && + usb_endpoint_dir_in(&tq->endp->desc))) { + spin_lock_irqsave(&tq->urb_lock, flags); + if (!list_empty(&tq->endp->urb_list)) { + urb = list_first_entry(&tq->endp->urb_list, struct urb, urb_list); + vurb = urb->hcpriv; + pr_debug("t2bce_vhci: suspend tq pre: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=1 first_state=%x\n", + tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled, + vurb ? vurb->state : 0xffffffff); + } else { + pr_debug("t2bce_vhci: suspend tq pre: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=0\n", + tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled); + } + spin_unlock_irqrestore(&tq->urb_lock, flags); + } + + bce_vhci_transfer_queue_suspend_pause(tq); + + if (tq->endp_addr == 0x00 || (usb_endpoint_xfer_int(&tq->endp->desc) && + usb_endpoint_dir_in(&tq->endp->desc))) { + spin_lock_irqsave(&tq->urb_lock, flags); + if (!list_empty(&tq->endp->urb_list)) { + urb = list_first_entry(&tq->endp->urb_list, struct urb, urb_list); + vurb = urb->hcpriv; + pr_debug("t2bce_vhci: suspend tq post: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=1 first_state=%x\n", + tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled, + vurb ? vurb->state : 0xffffffff); + } else { + pr_debug("t2bce_vhci: suspend tq post: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=0\n", + tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled); + } + spin_unlock_irqrestore(&tq->urb_lock, flags); + } + } + } + + /* + * Only occupied virtual ports are suspended. bridgeOS rejects holes in + * the virtual hub, and that rejection would abort system PM. + */ + pr_debug("t2bce_vhci: suspend occupied ports\n"); + for (i = 1; i < ARRAY_SIZE(vhci->port_to_device); i++) { + if (!vhci->port_to_device[i]) + continue; + + status = bce_vhci_cmd_port_suspend(&vhci->cq, (u8)i); + pr_debug("t2bce_vhci: suspend port %d -> %d\n", i, status); + if (status) + return status; + } + + /* Keep Linux-side event queues quiet before the BCE state-save transition. */ + bce_vhci_event_queue_pause(&vhci->ev_commands); + bce_vhci_event_queue_pause(&vhci->ev_system); + bce_vhci_event_queue_pause(&vhci->ev_isochronous); + bce_vhci_event_queue_pause(&vhci->ev_interrupt); + bce_vhci_event_queue_pause(&vhci->ev_asynchronous); + pr_debug("t2bce_vhci: suspend done\n"); + return 0; +} + +static int bce_vhci_suspend_prepare(struct usb_hcd *hcd) +{ + /* + * VHCI must quiesce usbcore-visible state before the core BCE mailbox + * suspend command asks bridgeOS to save or discard queue state. + */ + return bce_vhci_suspend_quiesce(hcd); +} + +static int bce_vhci_bus_suspend(struct usb_hcd *hcd) +{ + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + + pr_debug("t2bce_vhci: bus_suspend entry\n"); + vhci->port_change_pending = 0; + vhci->system_suspending = true; + status = bce_vhci_suspend_prepare(hcd); + if (status) + vhci->system_suspending = false; + pr_info("t2bce_vhci: bus_suspend exit status=%d\n", status); + return status; +} + +static int bce_vhci_resume_no_state(struct usb_hcd *hcd) +{ + int i; + int status; + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + + bce_vhci_event_queue_resume(&vhci->ev_system); + bce_vhci_event_queue_resume(&vhci->ev_isochronous); + bce_vhci_event_queue_resume(&vhci->ev_interrupt); + bce_vhci_event_queue_resume(&vhci->ev_asynchronous); + bce_vhci_event_queue_resume(&vhci->ev_commands); + + pr_debug("t2bce_vhci: no-state resume, rebuilding controller state\n"); + vhci->stateful_resume = false; + bce_vhci_forget_devices(vhci); + status = bce_vhci_start_controller(vhci); + if (status) { + pr_info("t2bce_vhci: no-state resume exit status=%d\n", status); + return status; + } + + pr_debug("t2bce_vhci: no-state resume, powering off all ports\n"); + for (i = 1; i <= vhci->port_count; i++) { + status = bce_vhci_cmd_port_power_off(&vhci->cq, (u8) i); + pr_debug("t2bce_vhci: no-state resume, port %d power_off -> %d\n", i, status); + } + + msleep(100); + + pr_debug("t2bce_vhci: no-state resume, powering on all ports\n"); + for (i = 1; i <= vhci->port_count; i++) { + status = bce_vhci_cmd_port_power_on(&vhci->cq, (u8) i); + pr_debug("t2bce_vhci: no-state resume, port %d power_on -> %d\n", i, status); + } + + pr_debug("t2bce_vhci: no-state resume, notifying usbcore about lost power\n"); + usb_root_hub_lost_power(hcd->self.root_hub); + vhci->no_state_resume = false; + pr_info("t2bce_vhci: no-state resume exit status=%d\n", status); + return status; +} + +static int bce_vhci_resume_all_queues(struct bce_vhci *vhci) +{ + int devid; + int endp; + int status; + + for (devid = 1; devid < ARRAY_SIZE(vhci->devices); devid++) { + if (!vhci->devices[devid]) + continue; + + for (endp = 0; endp < 32; endp++) { + if (!(vhci->devices[devid]->tq_mask & BIT(endp))) + continue; + + if (vhci->devices[devid]->tq[endp].endp_addr == 0x00) + pr_debug("t2bce_vhci: resume_all EP0 pre dev=%d paused_by=%x state=%x active=%u\n", + devid, vhci->devices[devid]->tq[endp].paused_by, + vhci->devices[devid]->tq[endp].state, + vhci->devices[devid]->tq[endp].active); + status = bce_vhci_transfer_queue_resume(&vhci->devices[devid]->tq[endp], + BCE_VHCI_PAUSE_SUSPEND); + if (vhci->devices[devid]->tq[endp].endp_addr == 0x00) + pr_debug("t2bce_vhci: resume_all EP0 post dev=%d status=%d paused_by=%x state=%x active=%u\n", + devid, status, vhci->devices[devid]->tq[endp].paused_by, + vhci->devices[devid]->tq[endp].state, + vhci->devices[devid]->tq[endp].active); + if (status) { + pr_err("t2bce_vhci: stateful resume queue failed: dev=%d ep=%02x status=%d\n", + devid, vhci->devices[devid]->tq[endp].endp_addr, status); + return status; + } + } + } + + return 0; +} + +static int bce_vhci_resume_stateful(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + + /* Linux preserved-state resume path. This is kept separate from the + * no-state rebuild path below. */ + vhci->port_change_pending = 0; + vhci->stateful_suspended_bounce = 0; + vhci->stateful_resume = true; + + bce_vhci_event_queue_resume(&vhci->ev_system); + bce_vhci_event_queue_resume(&vhci->ev_isochronous); + bce_vhci_event_queue_resume(&vhci->ev_interrupt); + bce_vhci_event_queue_resume(&vhci->ev_asynchronous); + + pr_debug("t2bce_vhci: stateful resume queues before firmware command events\n"); + status = bce_vhci_resume_all_queues(vhci); + pr_debug("t2bce_vhci: stateful resume enabling firmware command events status=%d\n", status); + bce_vhci_event_queue_resume(&vhci->ev_commands); + pr_info("t2bce_vhci: stateful resume exit status=%d\n", status); + return status; +} + +static int bce_vhci_bus_resume(struct usb_hcd *hcd) +{ + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + int status; + + pr_debug("t2bce_vhci: bus_resume entry no_state_resume=%d\n", vhci->no_state_resume); + vhci->port_change_pending = 0; + vhci->system_suspending = false; + pr_debug("t2bce_vhci: resume started\n"); + + if (vhci->no_state_resume) + status = bce_vhci_resume_no_state(hcd); + else + status = bce_vhci_resume_stateful(hcd); + + pr_info("t2bce_vhci: bus_resume exit status=%d no_state_resume=%d\n", status, vhci->no_state_resume); + return status; +} + +static int bce_vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags) +{ + struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; + if (!q) + return -ENOENT; + pr_debug("t2bce_vhci: urb_enqueue dev=%u ep=%02x\n", + q->dev_addr, urb->ep->desc.bEndpointAddress); + return bce_vhci_urb_create(q, urb); +} + +static int bce_vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status) +{ + struct bce_vhci_transfer_queue *q = urb->ep->hcpriv; + if (!q) + return -ENOENT; + pr_debug("t2bce_vhci: urb_dequeue ep=%02x\n", urb->ep->desc.bEndpointAddress); + return bce_vhci_urb_request_cancel(q, urb, status); +} + +static void bce_vhci_endpoint_reset(struct usb_hcd *hcd, struct usb_host_endpoint *ep) +{ + struct bce_vhci_transfer_queue *q = ep->hcpriv; + pr_debug("t2bce_vhci: endpoint_reset\n"); + if (q) + bce_vhci_transfer_queue_request_reset(q); +} + +static u8 bce_vhci_endpoint_index(u8 addr) +{ + if (addr & 0x80) + return (u8) (0x10 + (addr & 0xf)); + return (u8) (addr & 0xf); +} + +static int bce_vhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) +{ + u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; + struct bce_vhci_device *vdev = vhci->devices[devid]; + pr_debug("t2bce_vhci: add_endpoint port=%u dev=%u ep_index=%u\n", + udev->portnum, devid, endp_index); + + if (udev->bus->root_hub == udev) /* The USB hub */ + return 0; + if (vdev == NULL) + return -ENODEV; + if (vdev->tq_mask & BIT(endp_index)) { + endp->hcpriv = &vdev->tq[endp_index]; + return 0; + } + + bce_vhci_create_transfer_queue(vhci, &vdev->tq[endp_index], endp, devid, + usb_endpoint_dir_in(&endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE); + endp->hcpriv = &vdev->tq[endp_index]; + vdev->tq_mask |= BIT(endp_index); + + bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &endp->desc); + return 0; +} + +static int bce_vhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp) +{ + u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress); + struct bce_vhci *vhci = bce_vhci_from_hcd(hcd); + bce_vhci_device_t devid = vhci->port_to_device[udev->portnum]; + struct bce_vhci_transfer_queue *q = endp->hcpriv; + struct bce_vhci_device *vdev; + pr_debug("t2bce_vhci: drop_endpoint port=%u ep_index=%u\n", udev->portnum, endp_index); + if (!devid || !vhci->devices[devid]) { + endp->hcpriv = NULL; + return 0; + } + vdev = vhci->devices[devid]; + if (!q) { + if (vdev->tq_mask & BIT(endp_index)) { + pr_err("t2bce_vhci: endpoint missing hcpriv while still tracked\n"); + q = &vdev->tq[endp_index]; + } else { + return 0; + } + } + + bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) (endp->desc.bEndpointAddress & 0x8Fu)); + vdev->tq_mask &= ~BIT(endp_index); + bce_vhci_destroy_transfer_queue(vhci, q); + endp->hcpriv = NULL; + return 0; +} + +static int bce_vhci_create_message_queues(struct bce_vhci *vhci) +{ + if (bce_vhci_message_queue_create(vhci, &vhci->msg_commands, "VHC1HostCommands") || + bce_vhci_message_queue_create(vhci, &vhci->msg_system, "VHC1HostSystemEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_isochronous, "VHC1HostIsochronousEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_interrupt, "VHC1HostInterruptEvents") || + bce_vhci_message_queue_create(vhci, &vhci->msg_asynchronous, "VHC1HostAsynchronousEvents")) { + bce_vhci_destroy_message_queues(vhci); + return -EINVAL; + } + spin_lock_init(&vhci->msg_asynchronous_lock); + bce_vhci_command_queue_create(&vhci->cq, &vhci->msg_commands); + return 0; +} + +static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci) +{ + bce_vhci_command_queue_destroy(&vhci->cq); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_commands); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_system); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_isochronous); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_interrupt); + bce_vhci_message_queue_destroy(vhci, &vhci->msg_asynchronous); +} + +static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); +static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg); + +static int bce_vhci_create_event_queues(struct bce_vhci *vhci) +{ + vhci->ev_cq = t2bce_core_create_cq(vhci->client, 0x100); + if (!vhci->ev_cq) + return -EINVAL; +#define CREATE_EVENT_QUEUE(field, name, cb) bce_vhci_event_queue_create(vhci, &vhci->field, name, cb) + if (__bce_vhci_event_queue_create(vhci, &vhci->ev_commands, "VHC1FirmwareCommands", + bce_vhci_firmware_event_completion) || + CREATE_EVENT_QUEUE(ev_system, "VHC1FirmwareSystemEvents", bce_vhci_handle_system_event) || + CREATE_EVENT_QUEUE(ev_isochronous, "VHC1FirmwareIsochronousEvents", bce_vhci_handle_usb_event) || + CREATE_EVENT_QUEUE(ev_interrupt, "VHC1FirmwareInterruptEvents", bce_vhci_handle_usb_event) || + CREATE_EVENT_QUEUE(ev_asynchronous, "VHC1FirmwareAsynchronousEvents", bce_vhci_handle_usb_event)) { + bce_vhci_destroy_event_queues(vhci); + return -EINVAL; + } +#undef CREATE_EVENT_QUEUE + return 0; +} + +static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci) +{ + bce_vhci_event_queue_destroy(vhci, &vhci->ev_commands); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_system); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_isochronous); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_interrupt); + bce_vhci_event_queue_destroy(vhci, &vhci->ev_asynchronous); + if (vhci->ev_cq) + t2bce_core_destroy_cq(vhci->client, vhci->ev_cq); +} + +static void bce_vhci_send_fw_event_response(struct bce_vhci *vhci, struct bce_vhci_message *req, u16 status) +{ + unsigned long timeout = 1000; + struct bce_vhci_message r = *req; + r.cmd = (u16) (req->cmd | 0x8000u); + r.status = status; + r.param1 = req->param1; + r.param2 = 0; + + if (t2bce_core_reserve_submission(vhci->msg_system.sq, &timeout)) { + pr_err("t2bce_vhci: Cannot reserve submision for FW event reply\n"); + return; + } + bce_vhci_message_queue_write(&vhci->msg_system, &r); +} + +static int bce_vhci_handle_firmware_event(struct bce_vhci *vhci, struct bce_vhci_message *msg) +{ + unsigned long flags; + bce_vhci_device_t devid; + u8 endp; + struct bce_vhci_device *dev; + struct bce_vhci_transfer_queue *tq; + if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE || msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { + devid = (bce_vhci_device_t) (msg->param1 & 0xff); + endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); + dev = vhci->devices[devid]; + if (!dev || !(dev->tq_mask & BIT(endp))) + return BCE_VHCI_BAD_ARGUMENT; + tq = &dev->tq[endp]; + } + + if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE) { + if (msg->param2 == BCE_VHCI_ENDPOINT_ACTIVE) { + int ret = bce_vhci_transfer_queue_resume(tq, BCE_VHCI_PAUSE_FIRMWARE); + + pr_debug("t2bce_vhci: FW endpoint request ACTIVE dev=%u ep=%02x paused_by=%x state=%x active=%u ret=%d reply=%u\n", + devid, tq->endp_addr, tq->paused_by, tq->state, tq->active, ret, + ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS); + return ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS; + } else if (msg->param2 == BCE_VHCI_ENDPOINT_PAUSED) { + int ret = bce_vhci_transfer_queue_pause(tq, BCE_VHCI_PAUSE_FIRMWARE); + + pr_debug("t2bce_vhci: FW endpoint request PAUSED dev=%u ep=%02x paused_by=%x state=%x active=%u ret=%d reply=%u\n", + devid, tq->endp_addr, tq->paused_by, tq->state, tq->active, ret, + ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS); + return ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS; + } + return BCE_VHCI_BAD_ARGUMENT; + } else if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) { + if (msg->param2 == BCE_VHCI_ENDPOINT_STALLED) { + tq->state = msg->param2; + spin_lock_irqsave(&tq->urb_lock, flags); + tq->active = false; + tq->stalled = true; + spin_unlock_irqrestore(&tq->urb_lock, flags); + pr_warn("t2bce_vhci: firmware endpoint stalled dev=%u ep=%02x paused_by=%x; %s\n", + devid, tq->endp_addr, tq->paused_by, + tq->endp_addr == 0x00 ? "scheduling EP0 reset" : "leaving stalled"); + if (tq->endp_addr == 0x00) + bce_vhci_transfer_queue_request_reset(tq); + return BCE_VHCI_SUCCESS; + } + return BCE_VHCI_BAD_ARGUMENT; + } + pr_warn("t2bce_vhci: Unhandled firmware event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + return BCE_VHCI_BAD_ARGUMENT; +} + +static void bce_vhci_handle_firmware_events_w(struct work_struct *ws) +{ + size_t cnt = 0; + int result; + struct bce_vhci *vhci = container_of(ws, struct bce_vhci, w_fw_events); + struct t2bce_core_queue_sq *sq = vhci->ev_commands.sq; + struct t2bce_core_sq_completion_data *cq; + struct bce_vhci_message *msg, *msg2 = NULL; + + while (true) { + if (msg2) { + msg = msg2; + msg2 = NULL; + } else if ((cq = t2bce_core_next_completion(sq))) { + if (cq->status == T2BCE_COMPLETION_ABORTED) { + t2bce_core_notify_submission_complete(sq); + continue; + } + msg = &vhci->ev_commands.data[t2bce_core_queue_sq_head(sq)]; + } else { + break; + } + + pr_debug("t2bce_vhci: Got fw event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2); + if ((cq = t2bce_core_next_completion(sq))) { + msg2 = &vhci->ev_commands.data[(t2bce_core_queue_sq_head(sq) + 1) % t2bce_core_queue_sq_capacity(sq)]; + pr_debug("t2bce_vhci: Got second fw event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + if (cq->status != T2BCE_COMPLETION_ABORTED && + msg2->cmd == (msg->cmd | 0x4000) && msg2->param1 == msg->param1) { + /* Take two elements */ + pr_debug("t2bce_vhci: Cancelled\n"); + bce_vhci_send_fw_event_response(vhci, msg, BCE_VHCI_ABORT); + + t2bce_core_notify_submission_complete(sq); + t2bce_core_notify_submission_complete(sq); + msg2 = NULL; + cnt += 2; + continue; + } + + pr_warn("t2bce_vhci: Handle fw event - unexpected cancellation\n"); + } + + result = bce_vhci_handle_firmware_event(vhci, msg); + bce_vhci_send_fw_event_response(vhci, msg, (u16) result); + + + t2bce_core_notify_submission_complete(sq); + ++cnt; + } + bce_vhci_event_queue_submit_pending(&vhci->ev_commands, cnt); + if (t2bce_core_queue_sq_available(sq) == t2bce_core_queue_sq_capacity(sq) - 1) { + pr_debug("t2bce_vhci: complete\n"); + complete(&vhci->ev_commands.queue_empty_completion); + } +} + +static void bce_vhci_firmware_event_completion(struct t2bce_core_queue_sq *sq) +{ + struct bce_vhci_event_queue *q = t2bce_core_queue_sq_userdata(sq); + queue_work(q->vhci->tq_state_wq, &q->vhci->w_fw_events); +} + +static const char *bce_vhci_port_state_change_label(u64 p2) +{ + return (p2 & 0x4) ? "connected" : "disconnected"; +} + +static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) +{ + if (msg->cmd & 0x8000) { + bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); + } else if (msg->cmd == BCE_VHCI_EV_PORT_STATUS_CHANGE && + msg->param1 > 0 && + msg->param1 < ARRAY_SIZE(q->vhci->port_to_device)) { + u32 normalized_status = (u32)msg->param2 & ~BCE_VHCI_PORT_STATUS_C_CONNECTION; + + if (q->vhci->stateful_resume && + (msg->param2 & BCE_VHCI_PORT_STATUS_CONNECTED) && + normalized_status == 0x285) { + set_bit(msg->param1 - 1, &q->vhci->stateful_suspended_bounce); + pr_debug("t2bce_vhci: marked stateful suspended bounce port=%u raw=%llx normalized=%x\n", + msg->param1, msg->param2, normalized_status); + } + set_bit(msg->param1 - 1, &q->vhci->port_change_pending); + pr_debug("t2bce_vhci: port state change event: port=%u state=%s status=%x p2=%llx\n", + msg->param1, bce_vhci_port_state_change_label(msg->param2), + msg->status, msg->param2); + } else { + pr_warn("t2bce_vhci: Unhandled system event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + } +} + +static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg) +{ + bce_vhci_device_t devid; + u8 endp; + struct bce_vhci_device *dev; + if (msg->cmd & 0x8000) { + bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg); + } else if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST || msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) { + devid = (bce_vhci_device_t) (msg->param1 & 0xff); + endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff)); + dev = q->vhci->devices[devid]; + if (!dev || (dev->tq_mask & BIT(endp)) == 0) { + pr_err("t2bce_vhci: Didn't find destination for transfer queue event\n"); + return; + } + bce_vhci_transfer_queue_event(&dev->tq[endp], msg); + } else { + pr_warn("t2bce_vhci: Unhandled USB event: %x s=%x p1=%x p2=%llx\n", + msg->cmd, msg->status, msg->param1, msg->param2); + } +} + + + +static const struct hc_driver bce_vhci_driver = { + .description = "t2bce_vhci", + .product_desc = "BCE VHCI Host Controller", + .hcd_priv_size = sizeof(struct bce_vhci *), + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0) + .flags = HCD_USB2, +#else + .flags = HCD_USB2 | HCD_DMA, +#endif + + .start = bce_vhci_start, + .stop = bce_vhci_stop, + .hub_status_data = bce_vhci_hub_status_data, + .hub_control = bce_vhci_hub_control, + .urb_enqueue = bce_vhci_urb_enqueue, + .urb_dequeue = bce_vhci_urb_dequeue, + .enable_device = bce_vhci_enable_device, + .free_dev = bce_vhci_free_device, + .address_device = bce_vhci_address_device, + .add_endpoint = bce_vhci_add_endpoint, + .drop_endpoint = bce_vhci_drop_endpoint, + .endpoint_reset = bce_vhci_endpoint_reset, + .check_bandwidth = bce_vhci_check_bandwidth, + .get_frame_number = bce_vhci_get_frame_number, + .bus_suspend = bce_vhci_bus_suspend, + .bus_resume = bce_vhci_bus_resume +}; + + +int __init bce_vhci_module_init(void) +{ + struct device *parent; + int result; + + if ((result = alloc_chrdev_region(&bce_vhci_chrdev, 0, 1, "t2bce_vhci"))) + goto fail_chrdev; +#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) + bce_vhci_class = class_create(THIS_MODULE, "t2bce_vhci"); +#else + bce_vhci_class = class_create("t2bce_vhci"); +#endif + if (IS_ERR(bce_vhci_class)) { + result = PTR_ERR(bce_vhci_class); + goto fail_class; + } + + global_vhci = kzalloc(sizeof(*global_vhci), GFP_KERNEL); + if (!global_vhci) { + result = -ENOMEM; + goto fail_alloc; + } + + parent = t2bce_core_device_get(); + if (IS_ERR(parent)) { + result = PTR_ERR(parent); + goto fail_create; + } + + result = bce_vhci_create(parent, global_vhci); + t2bce_core_device_put(parent); + if (result) + goto fail_create; + + pr_info("t2bce_vhci: module initialized\n"); + return 0; + +fail_create: + kfree(global_vhci); + global_vhci = NULL; +fail_alloc: + class_destroy(bce_vhci_class); +fail_class: + unregister_chrdev_region(bce_vhci_chrdev, 1); +fail_chrdev: + if (!result) + result = -EINVAL; + pr_info("t2bce_vhci: module init failed status=%d\n", result); + return result; +} +void __exit bce_vhci_module_exit(void) +{ + if (global_vhci) { + bce_vhci_destroy(global_vhci); + kfree(global_vhci); + global_vhci = NULL; + } + class_destroy(bce_vhci_class); + unregister_chrdev_region(bce_vhci_chrdev, 1); + pr_info("t2bce_vhci: module exited\n"); +} + +module_param_named(vhci_port_mask, bce_vhci_port_mask, ushort, 0444); +MODULE_PARM_DESC(vhci_port_mask, "Specifies which VHCI ports are enabled"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("AndrĂ© Eikmeyer "); +MODULE_DESCRIPTION("Apple T2 BCE VHCI Driver"); +MODULE_VERSION("0.02"); +MODULE_SOFTDEP("pre: t2bce_core"); +module_init(bce_vhci_module_init); +module_exit(bce_vhci_module_exit); diff --git a/drivers/staging/t2bce/t2bce_vhci/vhci.h b/drivers/staging/t2bce/t2bce_vhci/vhci.h new file mode 100644 index 000000000..cc18c6c58 --- /dev/null +++ b/drivers/staging/t2bce/t2bce_vhci/vhci.h @@ -0,0 +1,69 @@ +#ifndef BCE_VHCI_H +#define BCE_VHCI_H + +#include "queue.h" +#include "transfer.h" + +struct device; +struct usb_hcd; +struct t2bce_core_client; + +struct bce_vhci_device { + struct bce_vhci_transfer_queue tq[32]; + u32 tq_mask; +}; +struct bce_vhci { + struct t2bce_core_client *client; + dev_t vdevt; + struct device *vdev; + struct usb_hcd *hcd; + struct spinlock hcd_spinlock; + struct bce_vhci_message_queue msg_commands; + struct bce_vhci_message_queue msg_system; + struct bce_vhci_message_queue msg_isochronous; + struct bce_vhci_message_queue msg_interrupt; + struct bce_vhci_message_queue msg_asynchronous; + struct spinlock msg_asynchronous_lock; + struct bce_vhci_command_queue cq; + struct t2bce_core_queue_cq *ev_cq; + struct bce_vhci_event_queue ev_commands; + struct bce_vhci_event_queue ev_system; + struct bce_vhci_event_queue ev_isochronous; + struct bce_vhci_event_queue ev_interrupt; + struct bce_vhci_event_queue ev_asynchronous; + u16 port_mask; + u8 port_count; + u16 port_power_mask; + bce_vhci_device_t port_to_device[17]; + struct bce_vhci_device *devices[17]; + struct workqueue_struct *tq_state_wq; + struct work_struct w_fw_events; + struct work_struct w_add_hcd; + unsigned long port_change_pending; + unsigned long stateful_suspended_bounce; + bool stateful_resume; + bool no_state_resume; + bool hcd_registered; + bool system_suspending; +}; + +int __init bce_vhci_module_init(void); +void __exit bce_vhci_module_exit(void); + +int bce_vhci_create(struct device *parent, struct bce_vhci *vhci); +void bce_vhci_destroy(struct bce_vhci *vhci); +int bce_vhci_start(struct usb_hcd *hcd); +void bce_vhci_stop(struct usb_hcd *hcd); +int bce_vhci_add_hcd(struct bce_vhci *vhci); +void bce_vhci_remove_hcd(struct bce_vhci *vhci); +void bce_vhci_shutdown(struct bce_vhci *vhci); +void bce_vhci_pm_reset(struct bce_vhci *vhci); +void bce_vhci_pm_prepare(struct bce_vhci *vhci); +void bce_vhci_pm_prepare_no_state(struct bce_vhci *vhci); +void bce_vhci_pm_mark_no_state_resume(struct bce_vhci *vhci); +bool bce_vhci_pm_is_no_state_resume(struct bce_vhci *vhci); +void bce_vhci_pm_complete(struct bce_vhci *vhci); + +struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd); + +#endif //BCE_VHCI_H -- 2.54.0