imx.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * Driver for Motorola/Freescale IMX serial ports
 *
 * Based on drivers/char/serial.c, by Linus Torvalds, Theodore Ts'o.
 *
 * Author: Sascha Hauer <sascha@saschahauer.de>
 * Copyright (C) 2004 Pengutronix
 */

#include <linux/module.h>
#include <linux/ioport.h>
#include <linux/init.h>
#include <linux/console.h>
#include <linux/sysrq.h>
#include <linux/platform_device.h>
#include <linux/tty.h>
#include <linux/tty_flip.h>
#include <linux/serial_core.h>
#include <linux/serial.h>
#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/ktime.h>
#include <linux/pinctrl/consumer.h>
#include <linux/rational.h>
#include <linux/slab.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/io.h>
#include <linux/dma-mapping.h>

#include <asm/irq.h>
#include <linux/platform_data/dma-imx.h>

#include "serial_mctrl_gpio.h"

/* Register definitions */
#define URXD0 0x0  /* Receiver Register */
#define URTX0 0x40 /* Transmitter Register */
#define UCR1  0x80 /* Control Register 1 */
#define UCR2  0x84 /* Control Register 2 */
#define UCR3  0x88 /* Control Register 3 */
#define UCR4  0x8c /* Control Register 4 */
#define UFCR  0x90 /* FIFO Control Register */
#define USR1  0x94 /* Status Register 1 */
#define USR2  0x98 /* Status Register 2 */
#define UESC  0x9c /* Escape Character Register */
#define UTIM  0xa0 /* Escape Timer Register */
#define UBIR  0xa4 /* BRM Incremental Register */
#define UBMR  0xa8 /* BRM Modulator Register */
#define UBRC  0xac /* Baud Rate Count Register */
#define IMX21_ONEMS 0xb0 /* One Millisecond register */
#define IMX1_UTS 0xd0 /* UART Test Register on i.mx1 */
#define IMX21_UTS 0xb4 /* UART Test Register on all other i.mx*/

/* UART Control Register Bit Fields.*/
#define URXD_DUMMY_READ (1<<16)
#define URXD_CHARRDY	(1<<15)
#define URXD_ERR	(1<<14)
#define URXD_OVRRUN	(1<<13)
#define URXD_FRMERR	(1<<12)
#define URXD_BRK	(1<<11)
#define URXD_PRERR	(1<<10)
#define URXD_RX_DATA	(0xFF<<0)
#define UCR1_ADEN	(1<<15) /* Auto detect interrupt */
#define UCR1_ADBR	(1<<14) /* Auto detect baud rate */
#define UCR1_TRDYEN	(1<<13) /* Transmitter ready interrupt enable */
#define UCR1_IDEN	(1<<12) /* Idle condition interrupt */
#define UCR1_ICD_REG(x) (((x) & 3) << 10) /* idle condition detect */
#define UCR1_RRDYEN	(1<<9)	/* Recv ready interrupt enable */
#define UCR1_RXDMAEN	(1<<8)	/* Recv ready DMA enable */
#define UCR1_IREN	(1<<7)	/* Infrared interface enable */
#define UCR1_TXMPTYEN	(1<<6)	/* Transimitter empty interrupt enable */
#define UCR1_RTSDEN	(1<<5)	/* RTS delta interrupt enable */
#define UCR1_SNDBRK	(1<<4)	/* Send break */
#define UCR1_TXDMAEN	(1<<3)	/* Transmitter ready DMA enable */
#define IMX1_UCR1_UARTCLKEN (1<<2) /* UART clock enabled, i.mx1 only */
#define UCR1_ATDMAEN    (1<<2)  /* Aging DMA Timer Enable */
#define UCR1_DOZE	(1<<1)	/* Doze */
#define UCR1_UARTEN	(1<<0)	/* UART enabled */
#define UCR2_ESCI	(1<<15)	/* Escape seq interrupt enable */
#define UCR2_IRTS	(1<<14)	/* Ignore RTS pin */
#define UCR2_CTSC	(1<<13)	/* CTS pin control */
#define UCR2_CTS	(1<<12)	/* Clear to send */
#define UCR2_ESCEN	(1<<11)	/* Escape enable */
#define UCR2_PREN	(1<<8)	/* Parity enable */
#define UCR2_PROE	(1<<7)	/* Parity odd/even */
#define UCR2_STPB	(1<<6)	/* Stop */
#define UCR2_WS		(1<<5)	/* Word size */
#define UCR2_RTSEN	(1<<4)	/* Request to send interrupt enable */
#define UCR2_ATEN	(1<<3)	/* Aging Timer Enable */
#define UCR2_TXEN	(1<<2)	/* Transmitter enabled */
#define UCR2_RXEN	(1<<1)	/* Receiver enabled */
#define UCR2_SRST	(1<<0)	/* SW reset */
#define UCR3_DTREN	(1<<13) /* DTR interrupt enable */
#define UCR3_PARERREN	(1<<12) /* Parity enable */
#define UCR3_FRAERREN	(1<<11) /* Frame error interrupt enable */
#define UCR3_DSR	(1<<10) /* Data set ready */
#define UCR3_DCD	(1<<9)	/* Data carrier detect */
#define UCR3_RI		(1<<8)	/* Ring indicator */
#define UCR3_ADNIMP	(1<<7)	/* Autobaud Detection Not Improved */
#define UCR3_RXDSEN	(1<<6)	/* Receive status interrupt enable */
#define UCR3_AIRINTEN	(1<<5)	/* Async IR wake interrupt enable */
#define UCR3_AWAKEN	(1<<4)	/* Async wake interrupt enable */
#define UCR3_DTRDEN	(1<<3)	/* Data Terminal Ready Delta Enable. */
#define IMX21_UCR3_RXDMUXSEL	(1<<2)	/* RXD Muxed Input Select */
#define UCR3_INVT	(1<<1)	/* Inverted Infrared transmission */
#define UCR3_BPEN	(1<<0)	/* Preset registers enable */
#define UCR4_CTSTL_SHF	10	/* CTS trigger level shift */
#define UCR4_CTSTL_MASK	0x3F	/* CTS trigger is 6 bits wide */
#define UCR4_INVR	(1<<9)	/* Inverted infrared reception */
#define UCR4_ENIRI	(1<<8)	/* Serial infrared interrupt enable */
#define UCR4_WKEN	(1<<7)	/* Wake interrupt enable */
#define UCR4_REF16	(1<<6)	/* Ref freq 16 MHz */
#define UCR4_IDDMAEN    (1<<6)  /* DMA IDLE Condition Detected */
#define UCR4_IRSC	(1<<5)	/* IR special case */
#define UCR4_TCEN	(1<<3)	/* Transmit complete interrupt enable */
#define UCR4_BKEN	(1<<2)	/* Break condition interrupt enable */
#define UCR4_OREN	(1<<1)	/* Receiver overrun interrupt enable */
#define UCR4_DREN	(1<<0)	/* Recv data ready interrupt enable */
#define UFCR_RXTL_SHF	0	/* Receiver trigger level shift */
#define UFCR_DCEDTE	(1<<6)	/* DCE/DTE mode select */
#define UFCR_RFDIV	(7<<7)	/* Reference freq divider mask */
#define UFCR_RFDIV_REG(x)	(((x) < 7 ? 6 - (x) : 6) << 7)
#define UFCR_TXTL_SHF	10	/* Transmitter trigger level shift */
#define USR1_PARITYERR	(1<<15) /* Parity error interrupt flag */
#define USR1_RTSS	(1<<14) /* RTS pin status */
#define USR1_TRDY	(1<<13) /* Transmitter ready interrupt/dma flag */
#define USR1_RTSD	(1<<12) /* RTS delta */
#define USR1_ESCF	(1<<11) /* Escape seq interrupt flag */
#define USR1_FRAMERR	(1<<10) /* Frame error interrupt flag */
#define USR1_RRDY	(1<<9)	 /* Receiver ready interrupt/dma flag */
#define USR1_AGTIM	(1<<8)	 /* Ageing timer interrupt flag */
#define USR1_DTRD	(1<<7)	 /* DTR Delta */
#define USR1_RXDS	 (1<<6)	 /* Receiver idle interrupt flag */
#define USR1_AIRINT	 (1<<5)	 /* Async IR wake interrupt flag */
#define USR1_AWAKE	 (1<<4)	 /* Aysnc wake interrupt flag */
#define USR2_ADET	 (1<<15) /* Auto baud rate detect complete */
#define USR2_TXFE	 (1<<14) /* Transmit buffer FIFO empty */
#define USR2_DTRF	 (1<<13) /* DTR edge interrupt flag */
#define USR2_IDLE	 (1<<12) /* Idle condition */
#define USR2_RIDELT	 (1<<10) /* Ring Interrupt Delta */
#define USR2_RIIN	 (1<<9)	 /* Ring Indicator Input */
#define USR2_IRINT	 (1<<8)	 /* Serial infrared interrupt flag */
#define USR2_WAKE	 (1<<7)	 /* Wake */
#define USR2_DCDIN	 (1<<5)	 /* Data Carrier Detect Input */
#define USR2_RTSF	 (1<<4)	 /* RTS edge interrupt flag */
#define USR2_TXDC	 (1<<3)	 /* Transmitter complete */
#define USR2_BRCD	 (1<<2)	 /* Break condition */
#define USR2_ORE	(1<<1)	 /* Overrun error */
#define USR2_RDR	(1<<0)	 /* Recv data ready */
#define UTS_FRCPERR	(1<<13) /* Force parity error */
#define UTS_LOOP	(1<<12)	 /* Loop tx and rx */
#define UTS_TXEMPTY	 (1<<6)	 /* TxFIFO empty */
#define UTS_RXEMPTY	 (1<<5)	 /* RxFIFO empty */
#define UTS_TXFULL	 (1<<4)	 /* TxFIFO full */
#define UTS_RXFULL	 (1<<3)	 /* RxFIFO full */
#define UTS_SOFTRST	 (1<<0)	 /* Software reset */

/* We've been assigned a range on the "Low-density serial ports" major */
#define SERIAL_IMX_MAJOR	207
#define MINOR_START		16
#define DEV_NAME		"ttymxc"

/*
 * This determines how often we check the modem status signals
 * for any change.  They generally aren't connected to an IRQ
 * so we have to poll them.  We also check immediately before
 * filling the TX fifo incase CTS has been dropped.
 */
#define MCTRL_TIMEOUT	(250*HZ/1000)

#define DRIVER_NAME "IMX-uart"

#define UART_NR 8

/* i.MX21 type uart runs on all i.mx except i.MX1 and i.MX6q */
enum imx_uart_type {
	IMX1_UART,
	IMX21_UART,
	IMX53_UART,
	IMX6Q_UART,
};

/* device type dependent stuff */
struct imx_uart_data {
	unsigned uts_reg;
	enum imx_uart_type devtype;
};

enum imx_tx_state {
	OFF,
	WAIT_AFTER_RTS,
	SEND,
	WAIT_AFTER_SEND,
};

struct imx_port {
	struct uart_port	port;
	struct timer_list	timer;
	unsigned int		old_status;
	unsigned int		have_rtscts:1;
	unsigned int		have_rtsgpio:1;
	unsigned int		dte_mode:1;
	unsigned int		inverted_tx:1;
	unsigned int		inverted_rx:1;
	struct clk		*clk_ipg;
	struct clk		*clk_per;
	const struct imx_uart_data *devdata;

	struct mctrl_gpios *gpios;

	/* shadow registers */
	unsigned int ucr1;
	unsigned int ucr2;
	unsigned int ucr3;
	unsigned int ucr4;
	unsigned int ufcr;

	/* DMA fields */
	unsigned int		dma_is_enabled:1;
	unsigned int		dma_is_rxing:1;
	unsigned int		dma_is_txing:1;
	struct dma_chan		*dma_chan_rx, *dma_chan_tx;
	struct scatterlist	rx_sgl, tx_sgl[2];
	void			*rx_buf;
	struct circ_buf		rx_ring;
	unsigned int		rx_buf_size;
	unsigned int		rx_period_length;
	unsigned int		rx_periods;
	dma_cookie_t		rx_cookie;
	unsigned int		tx_bytes;
	unsigned int		dma_tx_nents;
	unsigned int            saved_reg[10];
	bool			context_saved;

	enum imx_tx_state	tx_state;
	struct hrtimer		trigger_start_tx;
	struct hrtimer		trigger_stop_tx;

	unsigned int		parity;
};

struct imx_port_ucrs {
	unsigned int	ucr1;
	unsigned int	ucr2;
	unsigned int	ucr3;
};

static struct imx_uart_data imx_uart_devdata[] = {
	[IMX1_UART] = {
		.uts_reg = IMX1_UTS,
		.devtype = IMX1_UART,
	},
	[IMX21_UART] = {
		.uts_reg = IMX21_UTS,
		.devtype = IMX21_UART,
	},
	[IMX53_UART] = {
		.uts_reg = IMX21_UTS,
		.devtype = IMX53_UART,
	},
	[IMX6Q_UART] = {
		.uts_reg = IMX21_UTS,
		.devtype = IMX6Q_UART,
	},
};

static const struct of_device_id imx_uart_dt_ids[] = {
	{ .compatible = "fsl,imx6q-uart", .data = &imx_uart_devdata[IMX6Q_UART], },
	{ .compatible = "fsl,imx53-uart", .data = &imx_uart_devdata[IMX53_UART], },
	{ .compatible = "fsl,imx1-uart", .data = &imx_uart_devdata[IMX1_UART], },
	{ .compatible = "fsl,imx21-uart", .data = &imx_uart_devdata[IMX21_UART], },
	{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, imx_uart_dt_ids);

static void imx_uart_writel(struct imx_port *sport, u32 val, u32 offset)
{
	switch (offset) {
	case UCR1:
		sport->ucr1 = val;
		break;
	case UCR2:
		sport->ucr2 = val;
		break;
	case UCR3:
		sport->ucr3 = val;
		break;
	case UCR4:
		sport->ucr4 = val;
		break;
	case UFCR:
		sport->ufcr = val;
		break;
	default:
		break;
	}
	writel(val, sport->port.membase + offset);
}

static u32 imx_uart_readl(struct imx_port *sport, u32 offset)
{
	switch (offset) {
	case UCR1:
		return sport->ucr1;
		break;
	case UCR2:
		/*
		 * UCR2_SRST is the only bit in the cached registers that might
		 * differ from the value that was last written. As it only
		 * automatically becomes one after being cleared, reread
		 * conditionally.
		 */
		if (!(sport->ucr2 & UCR2_SRST))
			sport->ucr2 = readl(sport->port.membase + offset);
		return sport->ucr2;
		break;
	case UCR3:
		return sport->ucr3;
		break;
	case UCR4:
		return sport->ucr4;
		break;
	case UFCR:
		return sport->ufcr;
		break;
	default:
		return readl(sport->port.membase + offset);
	}
}

static inline unsigned imx_uart_uts_reg(struct imx_port *sport)
{
	return sport->devdata->uts_reg;
}

static inline int imx_uart_is_imx1(struct imx_port *sport)
{
	return sport->devdata->devtype == IMX1_UART;
}

static inline int imx_uart_is_imx21(struct imx_port *sport)
{
	return sport->devdata->devtype == IMX21_UART;
}

static inline int imx_uart_is_imx53(struct imx_port *sport)
{
	return sport->devdata->devtype == IMX53_UART;
}

static inline int imx_uart_is_imx6q(struct imx_port *sport)
{
	return sport->devdata->devtype == IMX6Q_UART;
}
/*
 * Save and restore functions for UCR1, UCR2 and UCR3 registers
 */
#if IS_ENABLED(CONFIG_SERIAL_IMX_CONSOLE)
static void imx_uart_ucrs_save(struct imx_port *sport,
			       struct imx_port_ucrs *ucr)
{
	/* save control registers */
	ucr->ucr1 = imx_uart_readl(sport, UCR1);
	ucr->ucr2 = imx_uart_readl(sport, UCR2);
	ucr->ucr3 = imx_uart_readl(sport, UCR3);
}

static void imx_uart_ucrs_restore(struct imx_port *sport,
				  struct imx_port_ucrs *ucr)
{
	/* restore control registers */
	imx_uart_writel(sport, ucr->ucr1, UCR1);
	imx_uart_writel(sport, ucr->ucr2, UCR2);
	imx_uart_writel(sport, ucr->ucr3, UCR3);
}
#endif

/* called with port.lock taken and irqs caller dependent */
static void imx_uart_rts_active(struct imx_port *sport, u32 *ucr2)
{
	*ucr2 &= ~(UCR2_CTSC | UCR2_CTS);

	sport->port.mctrl |= TIOCM_RTS;
	mctrl_gpio_set(sport->gpios, sport->port.mctrl);
}

/* called with port.lock taken and irqs caller dependent */
static void imx_uart_rts_inactive(struct imx_port *sport, u32 *ucr2)
{
	*ucr2 &= ~UCR2_CTSC;
	*ucr2 |= UCR2_CTS;

	sport->port.mctrl &= ~TIOCM_RTS;
	mctrl_gpio_set(sport->gpios, sport->port.mctrl);
}

static void start_hrtimer_ms(struct hrtimer *hrt, unsigned long msec)
{
       hrtimer_start(hrt, ms_to_ktime(msec), HRTIMER_MODE_REL);
}

static inline void imx_uart_markspace_rx( struct imx_port *sport, unsigned int * rx_char)
{
	unsigned int rx = *rx_char;
	unsigned int markspace, byte_oe, controller_oe, error_oe, paritybit;

	if ( ! ( sport->parity & CMSPAR )) return;

	markspace = !!( sport->parity & UCR2_PROE ); // 1 for MARK, 0 for SPACE

	// As the controller works on ODD and EVEN, we need to
	// convert this to MARK and SPACE here
	// Count ones in the received byte, first bit is set if the number is odd
	// 0 for even, 1 for odd
	byte_oe = !!(__sw_hweight8(rx) & 1);
	controller_oe = !!(UCR2_PROE & imx_uart_readl(sport, UCR2));
	error_oe = !!( URXD_PRERR & rx );

	// Get the value of the parity bit
	// depends on the PRERR (parity error bit)
	//		 the configured parity mode PROE
	//		 and the number of ones in the data byte
	//		 The parity bit on the line is an xor of those
	paritybit = ( controller_oe ^ byte_oe ) ^ error_oe;

	// Check if the value of the parity bit is the expected
	if ( markspace == paritybit ){
		// Parity bit received has the expected value
		// Clear error bits
		rx &= ~URXD_PRERR;
		// if no other error bit is set, unset the ERR bit also
		if ( 0 == (rx & ( URXD_BRK | URXD_FRMERR | URXD_OVRRUN ) ))
			rx &= ~URXD_ERR;
	}else{
		// Parity bit received has not the expected value
		// Set error bits
		rx |= URXD_ERR | URXD_PRERR;
	}
	*rx_char = rx;
}

static irqreturn_t __imx_uart_rxint(int irq, void *dev_id);
inline void imx_uart_markspace_tx(struct imx_port *sport, unsigned char tx)
{
	unsigned int ucr2, byte_oe, markspace, controller_oe, needed_controller_oe;
    int ret;

	if ( ! ( sport->parity & CMSPAR )) return; 

	ucr2 = imx_uart_readl(sport, UCR2);

	byte_oe = !!(__sw_hweight8(tx) & 1);
	markspace = !!( sport->parity & UCR2_PROE ); // 1 for MARK, 0 for SPACE

	controller_oe = !!(UCR2_PROE & ucr2);

	// Find out which parity mode (ODD or EVEN) is needed to generate
	// the configured output value of the parity bit
	// markspace is the value we want.
	needed_controller_oe = byte_oe ^ markspace;

	if (needed_controller_oe != controller_oe ){ // Change the parity

		/* Wait until FIFO and shift registers are empty */
		while (!(imx_uart_readl(sport,USR2) & USR2_TXDC));

		/* Disable TX */
		ucr2 &= ~UCR2_TXEN;
		imx_uart_writel(sport, ucr2, UCR2);

		/* Disable RX, needs to be dissabled so the state of 
		   PROE can be matched exactly to the received bytes.
		   First wait readout the RX Fifo using the RX irq handler,
		   Disable the RX.
		   Read again to check if another byte has arrived.*/
		ret = __imx_uart_rxint(0, sport);
		ucr2 &= ~UCR2_RXEN;
		imx_uart_writel(sport, ucr2 , UCR2);
		ret = __imx_uart_rxint(0, sport);

		/* Reconfigure: Set parity and enable RX & TX again */
		if( needed_controller_oe ){
			ucr2 |= UCR2_PROE;
		}else{
			ucr2 &= ~UCR2_PROE;
		}
		ucr2 |= (UCR2_RXEN | UCR2_TXEN);
		imx_uart_writel(sport, ucr2, UCR2);
	}
}


/* called with port.lock taken and irqs off */
static void imx_uart_start_rx(struct uart_port *port)
{
	struct imx_port *sport = (struct imx_port *)port;
	unsigned int ucr1, ucr2;

	ucr1 = imx_uart_readl(sport, UCR1);
	ucr2 = imx_uart_readl(sport, UCR2);

	ucr2 |= UCR2_RXEN;

	if (sport->dma_is_enabled) {
		ucr1 |= UCR1_RXDMAEN | UCR1_ATDMAEN;
	} else {
		ucr1 |= UCR1_RRDYEN;
		ucr2 |= UCR2_ATEN;
	}

	/* Write UCR2 first as it includes RXEN */
	imx_uart_writel(sport, ucr2, UCR2);
	imx_uart_writel(sport, ucr1, UCR1);
}

/* called with port.lock taken and irqs off */
static void imx_uart_stop_tx(struct uart_port *port)
{
	struct imx_port *sport = (struct imx_port *)port;
	u32 ucr1, ucr4, usr2;

	if (sport->tx_state == OFF)
		return;

	/*
	 * We are maybe in the SMP context, so if the DMA TX thread is running
	 * on other cpu, we have to wait for it to finish.
	 */
	if (sport->dma_is_txing)
		return;

	ucr1 = imx_uart_readl(sport, UCR1);
	imx_uart_writel(sport, ucr1 & ~UCR1_TRDYEN, UCR1);

	usr2 = imx_uart_readl(sport, USR2);
	if (!(usr2 & USR2_TXDC)) {
		/* The shifter is still busy, so retry once TC triggers */
		return;
	}

	ucr4 = imx_uart_readl(sport, UCR4);
	ucr4 &= ~UCR4_TCEN;
	imx_uart_writel(sport, ucr4, UCR4);

	/* in rs485 mode disable transmitter */
	if (port->rs485.flags & SER_RS485_ENABLED) {
		if (sport->tx_state == SEND) {
			sport->tx_state = WAIT_AFTER_SEND;
			start_hrtimer_ms(&sport->trigger_stop_tx,
					 port->rs485.delay_rts_after_send);
			return;
		}

		if (sport->tx_state == WAIT_AFTER_RTS ||
		    sport->tx_state == WAIT_AFTER_SEND) {
			u32 ucr2;

			hrtimer_try_to_cancel(&sport->trigger_start_tx);

			ucr2 = imx_uart_readl(sport, UCR2);
			if (port->rs485.flags & SER_RS485_RTS_AFTER_SEND)
				imx_uart_rts_active(sport, &ucr2);
			else
				imx_uart_rts_inactive(sport, &ucr2);
			imx_uart_writel(sport, ucr2, UCR2);

			imx_uart_start_rx(port);

			sport->tx_state = OFF;
		}
	} else {
		sport->tx_state = OFF;
	}
}

/* called with port.lock taken and irqs off */
static void imx_uart_stop_rx(struct uart_port *port)
{
	struct imx_port *sport = (struct imx_port *)port;
	u32 ucr1, ucr2, ucr4;

	ucr1 = imx_uart_readl(sport, UCR1);
	ucr2 = imx_uart_readl(sport, UCR2);
	ucr4 = imx_uart_readl(sport, UCR4);

	if (sport->dma_is_enabled) {
		ucr1 &= ~(UCR1_RXDMAEN | UCR1_ATDMAEN);
	} else {
		ucr1 &= ~UCR1_RRDYEN;
		ucr2 &= ~UCR2_ATEN;
		ucr4 &= ~UCR4_OREN;
	}
	imx_uart_writel(sport, ucr1, UCR1);
	imx_uart_writel(sport, ucr4, UCR4);

	ucr2 &= ~UCR2_RXEN;
	imx_uart_writel(sport, ucr2, UCR2);
}

/* called with port.lock taken and irqs off */
static void imx_uart_enable_ms(struct uart_port *port)
{
	struct imx_port *sport = (struct imx_port *)port;

	mod_timer(&sport->timer, jiffies);

	mctrl_gpio_enable_ms(sport->gpios);
}

static void imx_uart_dma_tx(struct imx_port *sport);

/* called with port.lock taken and irqs off */
static inline void imx_uart_transmit_buffer(struct imx_port *sport)
{
	struct circ_buf *xmit = &sport->port.state->xmit;

	if (sport->port.x_char) {
		/* Send next char */
		imx_uart_writel(sport, sport->port.x_char, URTX0);
		sport->port.icount.tx++;
		sport->port.x_char = 0;
		return;
	}

	if (uart_circ_empty(xmit) || uart_tx_stopped(&sport->port)) {
		imx_uart_stop_tx(&sport->port);
		return;
	}

	if (sport->dma_is_enabled) {
		u32 ucr1;
		/*
		 * We've just sent a X-char Ensure the TX DMA is enabled
		 * and the TX IRQ is disabled.
		 **/
		ucr1 = imx_uart_readl(sport, UCR1);
		ucr1 &= ~UCR1_TRDYEN;
		if (sport->dma_is_txing) {
			ucr1 |= UCR1_TXDMAEN;
			imx_uart_writel(sport, ucr1, UCR1);
		} else {
			imx_uart_writel(sport, ucr1, UCR1);
			imx_uart_dma_tx(sport);
		}

		return;
	}

	while (!uart_circ_empty(xmit) &&
	       !(imx_uart_readl(sport, imx_uart_uts_reg(sport)) & UTS_TXFULL)) {

		/* Special handling for MARK and SPACE parity */
		imx_uart_markspace_tx( sport, xmit->buf[xmit->tail]);

		/* send xmit->buf[xmit->tail]
		 * out the port here */
		imx_uart_writel(sport, xmit->buf[xmit->tail], URTX0);
		xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
		sport->port.icount.tx++;
	}

	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
		uart_write_wakeup(&sport->port);

	if (uart_circ_empty(xmit))
		imx_uart_stop_tx(&sport->port);
}

static void imx_uart_dma_tx_callback(void *data)
{
	struct imx_port *sport = data;
	struct scatterlist *sgl = &sport->tx_sgl[0];
	struct circ_buf *xmit = &sport->port.state->xmit;
	unsigned long flags;
	u32 ucr1;

	spin_lock_irqsave(&sport->port.lock, flags);

	dma_unmap_sg(sport->port.dev, sgl, sport->dma_tx_nents, DMA_TO_DEVICE);

	ucr1 = imx_uart_readl(sport, UCR1);
	ucr1 &= ~UCR1_TXDMAEN;
	imx_uart_writel(sport, ucr1, UCR1);

	/* update the stat */
	xmit->tail = (xmit->tail + sport->tx_bytes) & (UART_XMIT_SIZE - 1);
	sport->port.icount.tx += sport->tx_bytes;

	dev_dbg(sport->port.dev, "we finish the TX DMA.\n");

	sport->dma_is_txing = 0;

	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
		uart_write_wakeup(&sport->port);

	if (!uart_circ_empty(xmit) && !uart_tx_stopped(&sport->port))
		imx_uart_dma_tx(sport);
	else if (sport->port.rs485.flags & SER_RS485_ENABLED) {
		u32 ucr4 = imx_uart_readl(sport, UCR4);
		ucr4 |= UCR4_TCEN;
		imx_uart_writel(sport, ucr4, UCR4);
	}

	spin_unlock_irqrestore(&sport->port.lock, flags);
}

/* called with port.lock taken and irqs off */
static void imx_uart_dma_tx(struct imx_port *sport)
{
	struct circ_buf *xmit = &sport->port.state->xmit;
	struct scatterlist *sgl = sport->tx_sgl;
	struct dma_async_tx_descriptor *desc;
	struct dma_chan	*chan = sport->dma_chan_tx;
	struct device *dev = sport->port.dev;
	u32 ucr1, ucr4;
	int ret;

	if (sport->dma_is_txing)
		return;

	ucr4 = imx_uart_readl(sport, UCR4);
	ucr4 &= ~UCR4_TCEN;
	imx_uart_writel(sport, ucr4, UCR4);

	sport->tx_bytes = uart_circ_chars_pending(xmit);

	if (xmit->tail < xmit->head || xmit->head == 0) {
		sport->dma_tx_nents = 1;
		sg_init_one(sgl, xmit->buf + xmit->tail, sport->tx_bytes);
	} else {
		sport->dma_tx_nents = 2;
		sg_init_table(sgl, 2);
		sg_set_buf(sgl, xmit->buf + xmit->tail,
				UART_XMIT_SIZE - xmit->tail);
		sg_set_buf(sgl + 1, xmit->buf, xmit->head);
	}

	ret = dma_map_sg(dev, sgl, sport->dma_tx_nents, DMA_TO_DEVICE);
	if (ret == 0) {
		dev_err(dev, "DMA mapping error for TX.\n");
		return;
	}
	desc = dmaengine_prep_slave_sg(chan, sgl, ret,
					DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT);
	if (!desc) {
		dma_unmap_sg(dev, sgl, sport->dma_tx_nents,
			     DMA_TO_DEVICE);
		dev_err(dev, "We cannot prepare for the TX slave dma!\n");
		return;
	}
	desc->callback = imx_uart_dma_tx_callback;
	desc->callback_param = sport;

	dev_dbg(dev, "TX: prepare to send %lu bytes by DMA.\n",
			uart_circ_chars_pending(xmit));

	ucr1 = imx_uart_readl(sport, UCR1);
	ucr1 |= UCR1_TXDMAEN;
	imx_uart_writel(sport, ucr1, UCR1);

	/* fire it */
	sport->dma_is_txing = 1;
	dmaengine_submit(desc);
	dma_async_issue_pending(chan);
	return;
}

/* called with port.lock taken and irqs off */
static void imx_uart_start_tx(struct uart_port *port)
{
	struct imx_port *sport = (struct imx_port *)port;
	u32 ucr1;

	if (!sport->port.x_char && uart_circ_empty(&port->state->xmit))
		return;

	/*
	 * We cannot simply do nothing here if sport->tx_state == SEND already
	 * because UCR1_TXMPTYEN might already have been cleared in
	 * imx_uart_stop_tx(), but tx_state is still SEND.
	 */

	if (port->rs485.flags & SER_RS485_ENABLED) {
		if (sport->tx_state == OFF) {
			u32 ucr2 = imx_uart_readl(sport, UCR2);
			if (port->rs485.flags & SER_RS485_RTS_ON_SEND)
				imx_uart_rts_active(sport, &ucr2);
			else
				imx_uart_rts_inactive(sport, &ucr2);
			imx_uart_writel(sport, ucr2, UCR2);

			if (!(port->rs485.flags & SER_RS485_RX_DURING_TX))
				imx_uart_stop_rx(port);

			sport->tx_state = WAIT_AFTER_RTS;
			start_hrtimer_ms(&sport->trigger_start_tx,
					 port->rs485.delay_rts_before_send);
			return;
		}

		if (sport->tx_state == WAIT_AFTER_SEND
		    || sport->tx_state == WAIT_AFTER_RTS) {

			hrtimer_try_to_cancel(&sport->trigger_stop_tx);

			/*
			 * Enable transmitter and shifter empty irq only if DMA
			 * is off.  In the DMA case this is done in the
			 * tx-callback.
			 */
			if (!sport->dma_is_enabled) {
				u32 ucr4 = imx_uart_readl(sport, UCR4);
				ucr4 |= UCR4_TCEN;
				imx_uart_writel(sport, ucr4, UCR4);
			}

			sport->tx_state = SEND;
		}
	} else {
		sport->tx_state = SEND;
	}

	if (!sport->dma_is_enabled) {
		ucr1 = imx_uart_readl(sport, UCR1);
		imx_uart_writel(sport, ucr1 | UCR1_TRDYEN, UCR1);
	}

	if (sport->dma_is_enabled) {
		if (sport->port.x_char) {
			/* We have X-char to send, so enable TX IRQ and
			 * disable TX DMA to let TX interrupt to send X-char */
			ucr1 = imx_uart_readl(sport, UCR1);
			ucr1 &= ~UCR1_TXDMAEN;
			ucr1 |= UCR1_TRDYEN;
			imx_uart_writel(sport, ucr1, UCR1);
			return;
		}

		if (!uart_circ_empty(&port->state->xmit) &&
		    !uart_tx_stopped(port))
			imx_uart_dma_tx(sport);
		return;
	}
}

static irqreturn_t __imx_uart_rtsint(int irq, void *dev_id)
{
	struct imx_port *sport = dev_id;
	u32 usr1;

	imx_uart_writel(sport, USR1_RTSD, USR1);
	usr1 = imx_uart_readl(sport, USR1) & USR1_RTSS;
	uart_handle_cts_change(&sport->port, !!usr1);
	wake_up_interruptible(&sport->port.state->port.delta_msr_wait);

	return IRQ_HANDLED;
}

static irqreturn_t imx_uart_rtsint(int irq, void *dev_id)
{
	struct imx_port *sport = dev_id;
	irqreturn_t ret;

	spin_lock(&sport->port.lock);

	ret = __imx_uart_rtsint(irq, dev_id);

	spin_unlock(&sport->port.lock);

	return ret;
}

static irqreturn_t imx_uart_txint(int irq, void *dev_id)
{
	struct imx_port *sport = dev_id;

	spin_lock(&sport->port.lock);
	imx_uart_transmit_buffer(sport);
	spin_unlock(&sport->port.lock);
	return IRQ_HANDLED;
}

static irqreturn_t __imx_uart_rxint(int irq, void *dev_id)
{
	struct imx_port *sport = dev_id;
	unsigned int rx, flg, ignored = 0;
	struct tty_port *port = &sport->port.state->port;

	while (imx_uart_readl(sport, USR2) & USR2_RDR) {
		u32 usr2;

		flg = TTY_NORMAL;
		sport->port.icount.rx++;

		rx = imx_uart_readl(sport, URXD0);

		usr2 = imx_uart_readl(sport, USR2);
		if (usr2 & USR2_BRCD) {
			imx_uart_writel(sport, USR2_BRCD, USR2);
			if (uart_handle_break(&sport->port))
				continue;
		}

		if (uart_handle_sysrq_char(&sport->port, (unsigned char)rx))
			continue;

		imx_uart_markspace_rx( sport, &rx);

		if (unlikely(rx & URXD_ERR)) {
			if (rx & URXD_BRK)
				sport->port.icount.brk++;
			else if (rx & URXD_PRERR)
				sport->port.icount.parity++;
			else if (rx & URXD_FRMERR)
				sport->port.icount.frame++;
			if (rx & URXD_OVRRUN)
				sport->port.icount.overrun++;

			if (rx & sport->port.ignore_status_mask) {
				if (++ignored > 100)
					goto out;
				continue;
			}

			rx &= (sport->port.read_status_mask | 0xFF);

			if (rx & URXD_BRK)
				flg = TTY_BREAK;
			else if (rx & URXD_PRERR)
				flg = TTY_PARITY;
			else if (rx & URXD_FRMERR)
				flg = TTY_FRAME;
			if (rx & URXD_OVRRUN)
				flg = TTY_OVERRUN;

			sport->port.sysrq = 0;
		}

		if (sport->port.ignore_status_mask & URXD_DUMMY_READ)
			goto out;

		if (tty_insert_flip_char(port, rx, flg) == 0)
			sport->port.icount.buf_overrun++;
	}

out:
	tty_flip_buffer_push(port);

	return IRQ_HANDLED;
}

static irqreturn_t imx_uart_rxint(int irq, void *dev_id)
{
	struct imx_port *sport = dev_id;
	irqreturn_t ret;

	spin_lock(&sport->port.lock);

	ret = __imx_uart_rxint(irq, dev_id);

	spin_unlock(&sport->port.lock);

	return ret;
}

static void imx_uart_clear_rx_errors(struct imx_port *sport);

/*
 * We have a modem side uart, so the meanings of RTS and CTS are inverted.
 */
static unsigned int imx_uart_get_hwmctrl(struct imx_port *sport)
{
	unsigned int tmp = TIOCM_DSR;
	unsigned usr1 = imx_uart_readl(sport, USR1);
	unsigned usr2 = imx_uart_readl(sport, USR2);

	if (usr1 & USR1_RTSS)
		tmp |= TIOCM_CTS;

	/* in DCE mode DCDIN is always 0 */
	if (!(usr2 & USR2_DCDIN))
		tmp |= TIOCM_CAR;

	if (sport->dte_mode)
		if (!(imx_uart_readl(sport, USR2) & USR2_RIIN))
			tmp |= TIOCM_RI;

	return tmp;
}

/*
 * Handle any change of modem status signal since we were last called.
 */
static void imx_uart_mctrl_check(struct imx_port *sport)
{
	unsigned int status, changed;

	status = imx_uart_get_hwmctrl(sport);