drivers/net/ethernet/intel/ixgbe/ixgbe_ptp.c

   1 /*******************************************************************************
   2
   3   Intel 10 Gigabit PCI Express Linux driver
   4   Copyright(c) 1999 - 2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27 #include "ixgbe.h"
  28 #include <linux/export.h>
  29 #include <linux/ptp_classify.h>
  30
  31 /*
  32  * The 82599 and the X540 do not have true 64bit nanosecond scale
  33  * counter registers. Instead, SYSTIME is defined by a fixed point
  34  * system which allows the user to define the scale counter increment
  35  * value at every level change of the oscillator driving the SYSTIME
  36  * value. For both devices the TIMINCA:IV field defines this
  37  * increment. On the X540 device, 31 bits are provided. However on the
  38  * 82599 only provides 24 bits. The time unit is determined by the
  39  * clock frequency of the oscillator in combination with the TIMINCA
  40  * register. When these devices link at 10Gb the oscillator has a
  41  * period of 6.4ns. In order to convert the scale counter into
  42  * nanoseconds the cyclecounter and timecounter structures are
  43  * used. The SYSTIME registers need to be converted to ns values by use
  44  * of only a right shift (division by power of 2). The following math
  45  * determines the largest incvalue that will fit into the available
  46  * bits in the TIMINCA register.
  47  *
  48  * PeriodWidth: Number of bits to store the clock period
  49  * MaxWidth: The maximum width value of the TIMINCA register
  50  * Period: The clock period for the oscillator
  51  * round(): discard the fractional portion of the calculation
  52  *
  53  * Period * [ 2 ^ ( MaxWidth - PeriodWidth ) ]
  54  *
  55  * For the X540, MaxWidth is 31 bits, and the base period is 6.4 ns
  56  * For the 82599, MaxWidth is 24 bits, and the base period is 6.4 ns
  57  *
  58  * The period also changes based on the link speed:
  59  * At 10Gb link or no link, the period remains the same.
  60  * At 1Gb link, the period is multiplied by 10. (64ns)
  61  * At 100Mb link, the period is multiplied by 100. (640ns)
  62  *
  63  * The calculated value allows us to right shift the SYSTIME register
  64  * value in order to quickly convert it into a nanosecond clock,
  65  * while allowing for the maximum possible adjustment value.
  66  *
  67  * These diagrams are only for the 10Gb link period
  68  *
  69  *           SYSTIMEH            SYSTIMEL
  70  *       +--------------+  +--------------+
  71  * X540  |      32      |  | 1 | 3 |  28  |
  72  *       *--------------+  +--------------+
  73  *        \________ 36 bits ______/  fract
  74  *
  75  *       +--------------+  +--------------+
  76  * 82599 |      32      |  | 8 | 3 |  21  |
  77  *       *--------------+  +--------------+
  78  *        \________ 43 bits ______/  fract
  79  *
  80  * The 36 bit X540 SYSTIME overflows every
  81  *   2^36 * 10^-9 / 60 = 1.14 minutes or 69 seconds
  82  *
  83  * The 43 bit 82599 SYSTIME overflows every
  84  *   2^43 * 10^-9 / 3600 = 2.4 hours
  85  */
  86 #define IXGBE_INCVAL_10GB 0x66666666
  87 #define IXGBE_INCVAL_1GB  0x40000000
  88 #define IXGBE_INCVAL_100  0x50000000
  89
  90 #define IXGBE_INCVAL_SHIFT_10GB  28
  91 #define IXGBE_INCVAL_SHIFT_1GB   24
  92 #define IXGBE_INCVAL_SHIFT_100   21
  93
  94 #define IXGBE_INCVAL_SHIFT_82599 7
  95 #define IXGBE_INCPER_SHIFT_82599 24
  96 #define IXGBE_MAX_TIMEADJ_VALUE  0x7FFFFFFFFFFFFFFFULL
  97
  98 #define IXGBE_OVERFLOW_PERIOD    (HZ * 30)
  99
 100 #ifndef NSECS_PER_SEC
 101 #define NSECS_PER_SEC 1000000000ULL
 102 #endif
 103
 104 static struct sock_filter ptp_filter[] = {
 105         PTP_FILTER
 106 };
 107
 108 /**
 109  * ixgbe_ptp_read - read raw cycle counter (to be used by time counter)
 110  * @cc: the cyclecounter structure
 111  *
 112  * this function reads the cyclecounter registers and is called by the
 113  * cyclecounter structure used to construct a ns counter from the
 114  * arbitrary fixed point registers
 115  */
 116 static cycle_t ixgbe_ptp_read(const struct cyclecounter *cc)
 117 {
 118         struct ixgbe_adapter *adapter =
 119                 container_of(cc, struct ixgbe_adapter, cc);
 120         struct ixgbe_hw *hw = &adapter->hw;
 121         u64 stamp = 0;
 122
 123         stamp |= (u64)IXGBE_READ_REG(hw, IXGBE_SYSTIML);
 124         stamp |= (u64)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) << 32;
 125
 126         return stamp;
 127 }
 128
 129 /**
 130  * ixgbe_ptp_adjfreq
 131  * @ptp: the ptp clock structure
 132  * @ppb: parts per billion adjustment from base
 133  *
 134  * adjust the frequency of the ptp cycle counter by the
 135  * indicated ppb from the base frequency.
 136  */
 137 static int ixgbe_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
 138 {
 139         struct ixgbe_adapter *adapter =
 140                 container_of(ptp, struct ixgbe_adapter, ptp_caps);
 141         struct ixgbe_hw *hw = &adapter->hw;
 142         u64 freq;
 143         u32 diff, incval;
 144         int neg_adj = 0;
 145
 146         if (ppb < 0) {
 147                 neg_adj = 1;
 148                 ppb = -ppb;
 149         }
 150
 151         smp_mb();
 152         incval = ACCESS_ONCE(adapter->base_incval);
 153
 154         freq = incval;
 155         freq *= ppb;
 156         diff = div_u64(freq, 1000000000ULL);
 157
 158         incval = neg_adj ? (incval - diff) : (incval + diff);
 159
 160         switch (hw->mac.type) {
 161         case ixgbe_mac_X540:
 162                 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, incval);
 163                 break;
 164         case ixgbe_mac_82599EB:
 165                 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA,
 166                                 (1 << IXGBE_INCPER_SHIFT_82599) |
 167                                 incval);
 168                 break;
 169         default:
 170                 break;
 171         }
 172
 173         return 0;
 174 }
 175
 176 /**
 177  * ixgbe_ptp_adjtime
 178  * @ptp: the ptp clock structure
 179  * @delta: offset to adjust the cycle counter by
 180  *
 181  * adjust the timer by resetting the timecounter structure.
 182  */
 183 static int ixgbe_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
 184 {
 185         struct ixgbe_adapter *adapter =
 186                 container_of(ptp, struct ixgbe_adapter, ptp_caps);
 187         unsigned long flags;
 188         u64 now;
 189
 190         spin_lock_irqsave(&adapter->tmreg_lock, flags);
 191
 192         now = timecounter_read(&adapter->tc);
 193         now += delta;
 194
 195         /* reset the timecounter */
 196         timecounter_init(&adapter->tc,
 197                          &adapter->cc,
 198                          now);
 199
 200         spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 201         return 0;
 202 }
 203
 204 /**
 205  * ixgbe_ptp_gettime
 206  * @ptp: the ptp clock structure
 207  * @ts: timespec structure to hold the current time value
 208  *
 209  * read the timecounter and return the correct value on ns,
 210  * after converting it into a struct timespec.
 211  */
 212 static int ixgbe_ptp_gettime(struct ptp_clock_info *ptp, struct timespec *ts)
 213 {
 214         struct ixgbe_adapter *adapter =
 215                 container_of(ptp, struct ixgbe_adapter, ptp_caps);
 216         u64 ns;
 217         u32 remainder;
 218         unsigned long flags;
 219
 220         spin_lock_irqsave(&adapter->tmreg_lock, flags);
 221         ns = timecounter_read(&adapter->tc);
 222         spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 223
 224         ts->tv_sec = div_u64_rem(ns, 1000000000ULL, &remainder);
 225         ts->tv_nsec = remainder;
 226
 227         return 0;
 228 }
 229
 230 /**
 231  * ixgbe_ptp_settime
 232  * @ptp: the ptp clock structure
 233  * @ts: the timespec containing the new time for the cycle counter
 234  *
 235  * reset the timecounter to use a new base value instead of the kernel
 236  * wall timer value.
 237  */
 238 static int ixgbe_ptp_settime(struct ptp_clock_info *ptp,
 239                              const struct timespec *ts)
 240 {
 241         struct ixgbe_adapter *adapter =
 242                 container_of(ptp, struct ixgbe_adapter, ptp_caps);
 243         u64 ns;
 244         unsigned long flags;
 245
 246         ns = ts->tv_sec * 1000000000ULL;
 247         ns += ts->tv_nsec;
 248
 249         /* reset the timecounter */
 250         spin_lock_irqsave(&adapter->tmreg_lock, flags);
 251         timecounter_init(&adapter->tc, &adapter->cc, ns);
 252         spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 253
 254         return 0;
 255 }
 256
 257 /**
 258  * ixgbe_ptp_enable
 259  * @ptp: the ptp clock structure
 260  * @rq: the requested feature to change
 261  * @on: whether to enable or disable the feature
 262  *
 263  * enable (or disable) ancillary features of the phc subsystem.
 264  * our driver only supports the PPS feature on the X540
 265  */
 266 static int ixgbe_ptp_enable(struct ptp_clock_info *ptp,
 267                             struct ptp_clock_request *rq, int on)
 268 {
 269         struct ixgbe_adapter *adapter =
 270                 container_of(ptp, struct ixgbe_adapter, ptp_caps);
 271
 272         /**
 273          * When PPS is enabled, unmask the interrupt for the ClockOut
 274          * feature, so that the interrupt handler can send the PPS
 275          * event when the clock SDP triggers. Clear mask when PPS is
 276          * disabled
 277          */
 278         if (rq->type == PTP_CLK_REQ_PPS) {
 279                 switch (adapter->hw.mac.type) {
 280                 case ixgbe_mac_X540:
 281                         if (on)
 282                                 adapter->flags2 |= IXGBE_FLAG2_PTP_PPS_ENABLED;
 283                         else
 284                                 adapter->flags2 &=
 285                                         ~IXGBE_FLAG2_PTP_PPS_ENABLED;
 286                         return 0;
 287                 default:
 288                         break;
 289                 }
 290         }
 291
 292         return -ENOTSUPP;
 293 }
 294
 295 /**
 296  * ixgbe_ptp_check_pps_event
 297  * @adapter: the private adapter structure
 298  * @eicr: the interrupt cause register value
 299  *
 300  * This function is called by the interrupt routine when checking for
 301  * interrupts. It will check and handle a pps event.
 302  */
 303 void ixgbe_ptp_check_pps_event(struct ixgbe_adapter *adapter, u32 eicr)
 304 {
 305         struct ixgbe_hw *hw = &adapter->hw;
 306         struct ptp_clock_event event;
 307
 308         event.type = PTP_CLOCK_PPS;
 309
 310         /* Make sure ptp clock is valid, and PPS event enabled */
 311         if (!adapter->ptp_clock ||
 312             !(adapter->flags2 & IXGBE_FLAG2_PTP_PPS_ENABLED))
 313                 return;
 314
 315         if (unlikely(eicr & IXGBE_EICR_TIMESYNC)) {
 316                 switch (hw->mac.type) {
 317                 case ixgbe_mac_X540:
 318                         ptp_clock_event(adapter->ptp_clock, &event);
 319                         break;
 320                 default:
 321                         break;
 322                 }
 323         }
 324 }
 325
 326 /**
 327  * ixgbe_ptp_enable_sdp
 328  * @hw: the hardware private structure
 329  * @shift: the clock shift for calculating nanoseconds
 330  *
 331  * this function enables the clock out feature on the sdp0 for the
 332  * X540 device. It will create a 1second periodic output that can be
 333  * used as the PPS (via an interrupt).
 334  *
 335  * It calculates when the systime will be on an exact second, and then
 336  * aligns the start of the PPS signal to that value. The shift is
 337  * necessary because it can change based on the link speed.
 338  */
 339 static void ixgbe_ptp_enable_sdp(struct ixgbe_hw *hw, int shift)
 340 {
 341         u32 esdp, tsauxc, clktiml, clktimh, trgttiml, trgttimh;
 342         u64 clock_edge = 0;
 343         u32 rem;
 344
 345         switch (hw->mac.type) {
 346         case ixgbe_mac_X540:
 347                 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
 348
 349                 /*
 350                  * enable the SDP0 pin as output, and connected to the native
 351                  * function for Timesync (ClockOut)
 352                  */
 353                 esdp |= (IXGBE_ESDP_SDP0_DIR |
 354                          IXGBE_ESDP_SDP0_NATIVE);
 355
 356                 /*
 357                  * enable the Clock Out feature on SDP0, and allow interrupts
 358                  * to occur when the pin changes
 359                  */
 360                 tsauxc = (IXGBE_TSAUXC_EN_CLK |
 361                           IXGBE_TSAUXC_SYNCLK |
 362                           IXGBE_TSAUXC_SDP0_INT);
 363
 364                 /* clock period (or pulse length) */
 365                 clktiml = (u32)(NSECS_PER_SEC << shift);
 366                 clktimh = (u32)((NSECS_PER_SEC << shift) >> 32);
 367
 368                 clock_edge |= (u64)IXGBE_READ_REG(hw, IXGBE_SYSTIML);
 369                 clock_edge |= (u64)IXGBE_READ_REG(hw, IXGBE_SYSTIMH) << 32;
 370
 371                 /*
 372                  * account for the fact that we can't do u64 division
 373                  * with remainder, by converting the clock values into
 374                  * nanoseconds first
 375                  */
 376                 clock_edge >>= shift;
 377                 div_u64_rem(clock_edge, NSECS_PER_SEC, &rem);
 378                 clock_edge += (NSECS_PER_SEC - rem);
 379                 clock_edge <<= shift;
 380
 381                 /* specify the initial clock start time */
 382                 trgttiml = (u32)clock_edge;
 383                 trgttimh = (u32)(clock_edge >> 32);
 384
 385                 IXGBE_WRITE_REG(hw, IXGBE_CLKTIML, clktiml);
 386                 IXGBE_WRITE_REG(hw, IXGBE_CLKTIMH, clktimh);
 387                 IXGBE_WRITE_REG(hw, IXGBE_TRGTTIML0, trgttiml);
 388                 IXGBE_WRITE_REG(hw, IXGBE_TRGTTIMH0, trgttimh);
 389
 390                 IXGBE_WRITE_REG(hw, IXGBE_ESDP, esdp);
 391                 IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, tsauxc);
 392
 393                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_TIMESYNC);
 394                 break;
 395         default:
 396                 break;
 397         }
 398 }
 399
 400 /**
 401  * ixgbe_ptp_disable_sdp
 402  * @hw: the private hardware structure
 403  *
 404  * this function disables the auxiliary SDP clock out feature
 405  */
 406 static void ixgbe_ptp_disable_sdp(struct ixgbe_hw *hw)
 407 {
 408         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_TIMESYNC);
 409         IXGBE_WRITE_REG(hw, IXGBE_TSAUXC, 0);
 410 }
 411
 412 /**
 413  * ixgbe_ptp_overflow_check - delayed work to detect SYSTIME overflow
 414  * @work: structure containing information about this work task
 415  *
 416  * this work function is scheduled to continue reading the timecounter
 417  * in order to prevent missing when the system time registers wrap
 418  * around. This needs to be run approximately twice a minute when no
 419  * PTP activity is occurring.
 420  */
 421 void ixgbe_ptp_overflow_check(struct ixgbe_adapter *adapter)
 422 {
 423         unsigned long elapsed_jiffies = adapter->last_overflow_check - jiffies;
 424         struct timespec ts;
 425
 426         if ((adapter->flags2 & IXGBE_FLAG2_OVERFLOW_CHECK_ENABLED) &&
 427             (elapsed_jiffies >= IXGBE_OVERFLOW_PERIOD)) {
 428                 ixgbe_ptp_gettime(&adapter->ptp_caps, &ts);
 429                 adapter->last_overflow_check = jiffies;
 430         }
 431 }
 432
 433 /**
 434  * ixgbe_ptp_match - determine if this skb matches a ptp packet
 435  * @skb: pointer to the skb
 436  * @hwtstamp: pointer to the hwtstamp_config to check
 437  *
 438  * Determine whether the skb should have been timestamped, assuming the
 439  * hwtstamp was set via the hwtstamp ioctl. Returns non-zero when the packet
 440  * should have a timestamp waiting in the registers, and 0 otherwise.
 441  *
 442  * V1 packets have to check the version type to determine whether they are
 443  * correct. However, we can't directly access the data because it might be
 444  * fragmented in the SKB, in paged memory. In order to work around this, we
 445  * use skb_copy_bits which will properly copy the data whether it is in the
 446  * paged memory fragments or not. We have to copy the IP header as well as the
 447  * message type.
 448  */
 449 static int ixgbe_ptp_match(struct sk_buff *skb, int rx_filter)
 450 {
 451         struct iphdr iph;
 452         u8 msgtype;
 453         unsigned int type, offset;
 454
 455         if (rx_filter == HWTSTAMP_FILTER_NONE)
 456                 return 0;
 457
 458         type = sk_run_filter(skb, ptp_filter);
 459
 460         if (likely(rx_filter == HWTSTAMP_FILTER_PTP_V2_EVENT))
 461                 return type & PTP_CLASS_V2;
 462
 463         /* For the remaining cases actually check message type */
 464         switch (type) {
 465         case PTP_CLASS_V1_IPV4:
 466                 skb_copy_bits(skb, OFF_IHL, &iph, sizeof(iph));
 467                 offset = ETH_HLEN + (iph.ihl << 2) + UDP_HLEN + OFF_PTP_CONTROL;
 468                 break;
 469         case PTP_CLASS_V1_IPV6:
 470                 offset = OFF_PTP6 + OFF_PTP_CONTROL;
 471                 break;
 472         default:
 473                 /* other cases invalid or handled above */
 474                 return 0;
 475         }
 476
 477         /* Make sure our buffer is long enough */
 478         if (skb->len < offset)
 479                 return 0;
 480
 481         skb_copy_bits(skb, offset, &msgtype, sizeof(msgtype));
 482
 483         switch (rx_filter) {
 484         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 485                 return (msgtype == IXGBE_RXMTRL_V1_SYNC_MSG);
 486                 break;
 487         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 488                 return (msgtype == IXGBE_RXMTRL_V1_DELAY_REQ_MSG);
 489                 break;
 490         default:
 491                 return 0;
 492         }
 493 }
 494
 495 /**
 496  * ixgbe_ptp_tx_hwtstamp - utility function which checks for TX time stamp
 497  * @q_vector: structure containing interrupt and ring information
 498  * @skb: particular skb to send timestamp with
 499  *
 500  * if the timestamp is valid, we convert it into the timecounter ns
 501  * value, then store that result into the shhwtstamps structure which
 502  * is passed up the network stack
 503  */
 504 void ixgbe_ptp_tx_hwtstamp(struct ixgbe_q_vector *q_vector,
 505                            struct sk_buff *skb)
 506 {
 507         struct ixgbe_adapter *adapter;
 508         struct ixgbe_hw *hw;
 509         struct skb_shared_hwtstamps shhwtstamps;
 510         u64 regval = 0, ns;
 511         u32 tsynctxctl;
 512         unsigned long flags;
 513
 514         /* we cannot process timestamps on a ring without a q_vector */
 515         if (!q_vector || !q_vector->adapter)
 516                 return;
 517
 518         adapter = q_vector->adapter;
 519         hw = &adapter->hw;
 520
 521         tsynctxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
 522         regval |= (u64)IXGBE_READ_REG(hw, IXGBE_TXSTMPL);
 523         regval |= (u64)IXGBE_READ_REG(hw, IXGBE_TXSTMPH) << 32;
 524
 525         /*
 526          * if TX timestamp is not valid, exit after clearing the
 527          * timestamp registers
 528          */
 529         if (!(tsynctxctl & IXGBE_TSYNCTXCTL_VALID))
 530                 return;
 531
 532         spin_lock_irqsave(&adapter->tmreg_lock, flags);
 533         ns = timecounter_cyc2time(&adapter->tc, regval);
 534         spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 535
 536         memset(&shhwtstamps, 0, sizeof(shhwtstamps));
 537         shhwtstamps.hwtstamp = ns_to_ktime(ns);
 538         skb_tstamp_tx(skb, &shhwtstamps);
 539 }
 540
 541 /**
 542  * ixgbe_ptp_rx_hwtstamp - utility function which checks for RX time stamp
 543  * @q_vector: structure containing interrupt and ring information
 544  * @rx_desc: the rx descriptor
 545  * @skb: particular skb to send timestamp with
 546  *
 547  * if the timestamp is valid, we convert it into the timecounter ns
 548  * value, then store that result into the shhwtstamps structure which
 549  * is passed up the network stack
 550  */
 551 void ixgbe_ptp_rx_hwtstamp(struct ixgbe_q_vector *q_vector,
 552                            union ixgbe_adv_rx_desc *rx_desc,
 553                            struct sk_buff *skb)
 554 {
 555         struct ixgbe_adapter *adapter;
 556         struct ixgbe_hw *hw;
 557         struct skb_shared_hwtstamps *shhwtstamps;
 558         u64 regval = 0, ns;
 559         u32 tsyncrxctl;
 560         unsigned long flags;
 561
 562         /* we cannot process timestamps on a ring without a q_vector */
 563         if (!q_vector || !q_vector->adapter)
 564                 return;
 565
 566         adapter = q_vector->adapter;
 567         hw = &adapter->hw;
 568
 569         tsyncrxctl = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
 570
 571         /* Check if we have a valid timestamp and make sure the skb should
 572          * have been timestamped */
 573         if (likely(!(tsyncrxctl & IXGBE_TSYNCRXCTL_VALID) ||
 574                    !ixgbe_ptp_match(skb, adapter->rx_hwtstamp_filter)))
 575                 return;
 576
 577         /*
 578          * Always read the registers, in order to clear a possible fault
 579          * because of stagnant RX timestamp values for a packet that never
 580          * reached the queue.
 581          */
 582         regval |= (u64)IXGBE_READ_REG(hw, IXGBE_RXSTMPL);
 583         regval |= (u64)IXGBE_READ_REG(hw, IXGBE_RXSTMPH) << 32;
 584
 585         /*
 586          * If the timestamp bit is set in the packet's descriptor, we know the
 587          * timestamp belongs to this packet. No other packet can be
 588          * timestamped until the registers for timestamping have been read.
 589          * Therefor only one packet with this bit can be in the queue at a
 590          * time, and the rx timestamp values that were in the registers belong
 591          * to this packet.
 592          *
 593          * If nothing went wrong, then it should have a skb_shared_tx that we
 594          * can turn into a skb_shared_hwtstamps.
 595          */
 596         if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_TS)))
 597                 return;
 598
 599         spin_lock_irqsave(&adapter->tmreg_lock, flags);
 600         ns = timecounter_cyc2time(&adapter->tc, regval);
 601         spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 602
 603         shhwtstamps = skb_hwtstamps(skb);
 604         shhwtstamps->hwtstamp = ns_to_ktime(ns);
 605 }
 606
 607 /**
 608  * ixgbe_ptp_hwtstamp_ioctl - control hardware time stamping
 609  * @adapter: pointer to adapter struct
 610  * @ifreq: ioctl data
 611  * @cmd: particular ioctl requested
 612  *
 613  * Outgoing time stamping can be enabled and disabled. Play nice and
 614  * disable it when requested, although it shouldn't case any overhead
 615  * when no packet needs it. At most one packet in the queue may be
 616  * marked for time stamping, otherwise it would be impossible to tell
 617  * for sure to which packet the hardware time stamp belongs.
 618  *
 619  * Incoming time stamping has to be configured via the hardware
 620  * filters. Not all combinations are supported, in particular event
 621  * type has to be specified. Matching the kind of event packet is
 622  * not supported, with the exception of "all V2 events regardless of
 623  * level 2 or 4".
 624  *
 625  * Since hardware always timestamps Path delay packets when timestamping V2
 626  * packets, regardless of the type specified in the register, only use V2
 627  * Event mode. This more accurately tells the user what the hardware is going
 628  * to do anyways.
 629  */
 630 int ixgbe_ptp_hwtstamp_ioctl(struct ixgbe_adapter *adapter,
 631                              struct ifreq *ifr, int cmd)
 632 {
 633         struct ixgbe_hw *hw = &adapter->hw;
 634         struct hwtstamp_config config;
 635         u32 tsync_tx_ctl = IXGBE_TSYNCTXCTL_ENABLED;
 636         u32 tsync_rx_ctl = IXGBE_TSYNCRXCTL_ENABLED;
 637         u32 tsync_rx_mtrl = 0;
 638         bool is_l4 = false;
 639         bool is_l2 = false;
 640         u32 regval;
 641
 642         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
 643                 return -EFAULT;
 644
 645         /* reserved for future extensions */
 646         if (config.flags)
 647                 return -EINVAL;
 648
 649         switch (config.tx_type) {
 650         case HWTSTAMP_TX_OFF:
 651                 tsync_tx_ctl = 0;
 652         case HWTSTAMP_TX_ON:
 653                 break;
 654         default:
 655                 return -ERANGE;
 656         }
 657
 658         switch (config.rx_filter) {
 659         case HWTSTAMP_FILTER_NONE:
 660                 tsync_rx_ctl = 0;
 661                 break;
 662         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 663                 tsync_rx_ctl |= IXGBE_TSYNCRXCTL_TYPE_L4_V1;
 664                 tsync_rx_mtrl = IXGBE_RXMTRL_V1_SYNC_MSG;
 665                 is_l4 = true;
 666                 break;
 667         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 668                 tsync_rx_ctl |= IXGBE_TSYNCRXCTL_TYPE_L4_V1;
 669                 tsync_rx_mtrl = IXGBE_RXMTRL_V1_DELAY_REQ_MSG;
 670                 is_l4 = true;
 671                 break;
 672         case HWTSTAMP_FILTER_PTP_V2_EVENT:
 673         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
 674         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 675         case HWTSTAMP_FILTER_PTP_V2_SYNC:
 676         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
 677         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 678         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 679         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 680         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 681                 tsync_rx_ctl |= IXGBE_TSYNCRXCTL_TYPE_EVENT_V2;
 682                 is_l2 = true;
 683                 is_l4 = true;
 684                 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 685                 break;
 686         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 687         case HWTSTAMP_FILTER_ALL:
 688         default:
 689                 /*
 690                  * register RXMTRL must be set in order to do V1 packets,
 691                  * therefore it is not possible to time stamp both V1 Sync and
 692                  * Delay_Req messages and hardware does not support
 693                  * timestamping all packets => return error
 694                  */
 695                 config.rx_filter = HWTSTAMP_FILTER_NONE;
 696                 return -ERANGE;
 697         }
 698
 699         if (hw->mac.type == ixgbe_mac_82598EB) {
 700                 if (tsync_rx_ctl | tsync_tx_ctl)
 701                         return -ERANGE;
 702                 return 0;
 703         }
 704
 705         /* Store filter value for later use */
 706         adapter->rx_hwtstamp_filter = config.rx_filter;
 707
 708         /* define ethertype filter for timestamped packets */
 709         if (is_l2)
 710                 IXGBE_WRITE_REG(hw, IXGBE_ETQF(3),
 711                                 (IXGBE_ETQF_FILTER_EN | /* enable filter */
 712                                  IXGBE_ETQF_1588 | /* enable timestamping */
 713                                  ETH_P_1588));     /* 1588 eth protocol type */
 714         else
 715                 IXGBE_WRITE_REG(hw, IXGBE_ETQF(3), 0);
 716
 717 #define PTP_PORT 319
 718         /* L4 Queue Filter[3]: filter by destination port and protocol */
 719         if (is_l4) {
 720                 u32 ftqf = (IXGBE_FTQF_PROTOCOL_UDP /* UDP */
 721                             | IXGBE_FTQF_POOL_MASK_EN /* Pool not compared */
 722                             | IXGBE_FTQF_QUEUE_ENABLE);
 723
 724                 ftqf |= ((IXGBE_FTQF_PROTOCOL_COMP_MASK /* protocol check */
 725                           & IXGBE_FTQF_DEST_PORT_MASK /* dest check */
 726                           & IXGBE_FTQF_SOURCE_PORT_MASK) /* source check */
 727                          << IXGBE_FTQF_5TUPLE_MASK_SHIFT);
 728
 729                 IXGBE_WRITE_REG(hw, IXGBE_L34T_IMIR(3),
 730                                 (3 << IXGBE_IMIR_RX_QUEUE_SHIFT_82599 |
 731                                  IXGBE_IMIR_SIZE_BP_82599));
 732
 733                 /* enable port check */
 734                 IXGBE_WRITE_REG(hw, IXGBE_SDPQF(3),
 735                                 (htons(PTP_PORT) |
 736                                  htons(PTP_PORT) << 16));
 737
 738                 IXGBE_WRITE_REG(hw, IXGBE_FTQF(3), ftqf);
 739
 740                 tsync_rx_mtrl |= PTP_PORT << 16;
 741         } else {
 742                 IXGBE_WRITE_REG(hw, IXGBE_FTQF(3), 0);
 743         }
 744
 745         /* enable/disable TX */
 746         regval = IXGBE_READ_REG(hw, IXGBE_TSYNCTXCTL);
 747         regval &= ~IXGBE_TSYNCTXCTL_ENABLED;
 748         regval |= tsync_tx_ctl;
 749         IXGBE_WRITE_REG(hw, IXGBE_TSYNCTXCTL, regval);
 750
 751         /* enable/disable RX */
 752         regval = IXGBE_READ_REG(hw, IXGBE_TSYNCRXCTL);
 753         regval &= ~(IXGBE_TSYNCRXCTL_ENABLED | IXGBE_TSYNCRXCTL_TYPE_MASK);
 754         regval |= tsync_rx_ctl;
 755         IXGBE_WRITE_REG(hw, IXGBE_TSYNCRXCTL, regval);
 756
 757         /* define which PTP packets are time stamped */
 758         IXGBE_WRITE_REG(hw, IXGBE_RXMTRL, tsync_rx_mtrl);
 759
 760         IXGBE_WRITE_FLUSH(hw);
 761
 762         /* clear TX/RX time stamp registers, just to be sure */
 763         regval = IXGBE_READ_REG(hw, IXGBE_TXSTMPH);
 764         regval = IXGBE_READ_REG(hw, IXGBE_RXSTMPH);
 765
 766         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
 767                 -EFAULT : 0;
 768 }
 769
 770 /**
 771  * ixgbe_ptp_start_cyclecounter - create the cycle counter from hw
 772  * @adapter: pointer to the adapter structure
 773  *
 774  * this function initializes the timecounter and cyclecounter
 775  * structures for use in generated a ns counter from the arbitrary
 776  * fixed point cycles registers in the hardware.
 777  *
 778  * A change in link speed impacts the frequency of the DMA clock on
 779  * the device, which is used to generate the cycle counter
 780  * registers. Therefor this function is called whenever the link speed
 781  * changes.
 782  *
 783  * This function also turns on the SDP pin for clock out feature (X540
 784  * only), because this is where the shift is first calculated.
 785  */
 786 void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter)
 787 {
 788         struct ixgbe_hw *hw = &adapter->hw;
 789         u32 incval = 0;
 790         u32 timinca = 0;
 791         u32 shift = 0;
 792         u32 cycle_speed;
 793         unsigned long flags;
 794
 795         /**
 796          * Determine what speed we need to set the cyclecounter
 797          * for. It should be different for 100Mb, 1Gb, and 10Gb. Treat
 798          * unknown speeds as 10Gb. (Hence why we can't just copy the
 799          * link_speed.
 800          */
 801         switch (adapter->link_speed) {
 802         case IXGBE_LINK_SPEED_100_FULL:
 803         case IXGBE_LINK_SPEED_1GB_FULL:
 804         case IXGBE_LINK_SPEED_10GB_FULL:
 805                 cycle_speed = adapter->link_speed;
 806                 break;
 807         default:
 808                 /* cycle speed should be 10Gb when there is no link */
 809                 cycle_speed = IXGBE_LINK_SPEED_10GB_FULL;
 810                 break;
 811         }
 812
 813         /*
 814          * grab the current TIMINCA value from the register so that it can be
 815          * double checked. If the register value has been cleared, it must be
 816          * reset to the correct value for generating a cyclecounter. If
 817          * TIMINCA is zero, the SYSTIME registers do not increment at all.
 818          */
 819         timinca = IXGBE_READ_REG(hw, IXGBE_TIMINCA);
 820
 821         /* Bail if the cycle speed didn't change and TIMINCA is non-zero */
 822         if (adapter->cycle_speed == cycle_speed && timinca)
 823                 return;
 824
 825         /* disable the SDP clock out */
 826         ixgbe_ptp_disable_sdp(hw);
 827
 828         /**
 829          * Scale the NIC cycle counter by a large factor so that
 830          * relatively small corrections to the frequency can be added
 831          * or subtracted. The drawbacks of a large factor include
 832          * (a) the clock register overflows more quickly, (b) the cycle
 833          * counter structure must be able to convert the systime value
 834          * to nanoseconds using only a multiplier and a right-shift,
 835          * and (c) the value must fit within the timinca register space
 836          * => math based on internal DMA clock rate and available bits
 837          */
 838         switch (cycle_speed) {
 839         case IXGBE_LINK_SPEED_100_FULL:
 840                 incval = IXGBE_INCVAL_100;
 841                 shift = IXGBE_INCVAL_SHIFT_100;
 842                 break;
 843         case IXGBE_LINK_SPEED_1GB_FULL:
 844                 incval = IXGBE_INCVAL_1GB;
 845                 shift = IXGBE_INCVAL_SHIFT_1GB;
 846                 break;
 847         case IXGBE_LINK_SPEED_10GB_FULL:
 848                 incval = IXGBE_INCVAL_10GB;
 849                 shift = IXGBE_INCVAL_SHIFT_10GB;
 850                 break;
 851         }
 852
 853         /**
 854          * Modify the calculated values to fit within the correct
 855          * number of bits specified by the hardware. The 82599 doesn't
 856          * have the same space as the X540, so bitshift the calculated
 857          * values to fit.
 858          */
 859         switch (hw->mac.type) {
 860         case ixgbe_mac_X540:
 861                 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA, incval);
 862                 break;
 863         case ixgbe_mac_82599EB:
 864                 incval >>= IXGBE_INCVAL_SHIFT_82599;
 865                 shift -= IXGBE_INCVAL_SHIFT_82599;
 866                 IXGBE_WRITE_REG(hw, IXGBE_TIMINCA,
 867                                 (1 << IXGBE_INCPER_SHIFT_82599) |
 868                                 incval);
 869                 break;
 870         default:
 871                 /* other devices aren't supported */
 872                 return;
 873         }
 874
 875         /* reset the system time registers */
 876         IXGBE_WRITE_REG(hw, IXGBE_SYSTIML, 0x00000000);
 877         IXGBE_WRITE_REG(hw, IXGBE_SYSTIMH, 0x00000000);
 878         IXGBE_WRITE_FLUSH(hw);
 879
 880         /* now that the shift has been calculated and the systime
 881          * registers reset, (re-)enable the Clock out feature*/
 882         ixgbe_ptp_enable_sdp(hw, shift);
 883
 884         /* store the new cycle speed */
 885         adapter->cycle_speed = cycle_speed;
 886
 887         ACCESS_ONCE(adapter->base_incval) = incval;
 888         smp_mb();
 889
 890         /* grab the ptp lock */
 891         spin_lock_irqsave(&adapter->tmreg_lock, flags);
 892
 893         memset(&adapter->cc, 0, sizeof(adapter->cc));
 894         adapter->cc.read = ixgbe_ptp_read;
 895         adapter->cc.mask = CLOCKSOURCE_MASK(64);
 896         adapter->cc.shift = shift;
 897         adapter->cc.mult = 1;
 898
 899         /* reset the ns time counter */
 900         timecounter_init(&adapter->tc, &adapter->cc,
 901                          ktime_to_ns(ktime_get_real()));
 902
 903         spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
 904 }
 905
 906 /**
 907  * ixgbe_ptp_init
 908  * @adapter: the ixgbe private adapter structure
 909  *
 910  * This function performs the required steps for enabling ptp
 911  * support. If ptp support has already been loaded it simply calls the
 912  * cyclecounter init routine and exits.
 913  */
 914 void ixgbe_ptp_init(struct ixgbe_adapter *adapter)
 915 {
 916         struct net_device *netdev = adapter->netdev;
 917
 918         switch (adapter->hw.mac.type) {
 919         case ixgbe_mac_X540:
 920                 snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
 921                 adapter->ptp_caps.owner = THIS_MODULE;
 922                 adapter->ptp_caps.max_adj = 250000000;
 923                 adapter->ptp_caps.n_alarm = 0;
 924                 adapter->ptp_caps.n_ext_ts = 0;
 925                 adapter->ptp_caps.n_per_out = 0;
 926                 adapter->ptp_caps.pps = 1;
 927                 adapter->ptp_caps.adjfreq = ixgbe_ptp_adjfreq;
 928                 adapter->ptp_caps.adjtime = ixgbe_ptp_adjtime;
 929                 adapter->ptp_caps.gettime = ixgbe_ptp_gettime;
 930                 adapter->ptp_caps.settime = ixgbe_ptp_settime;
 931                 adapter->ptp_caps.enable = ixgbe_ptp_enable;
 932                 break;
 933         case ixgbe_mac_82599EB:
 934                 snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr);
 935                 adapter->ptp_caps.owner = THIS_MODULE;
 936                 adapter->ptp_caps.max_adj = 250000000;
 937                 adapter->ptp_caps.n_alarm = 0;
 938                 adapter->ptp_caps.n_ext_ts = 0;
 939                 adapter->ptp_caps.n_per_out = 0;
 940                 adapter->ptp_caps.pps = 0;
 941                 adapter->ptp_caps.adjfreq = ixgbe_ptp_adjfreq;
 942                 adapter->ptp_caps.adjtime = ixgbe_ptp_adjtime;
 943                 adapter->ptp_caps.gettime = ixgbe_ptp_gettime;
 944                 adapter->ptp_caps.settime = ixgbe_ptp_settime;
 945                 adapter->ptp_caps.enable = ixgbe_ptp_enable;
 946                 break;
 947         default:
 948                 adapter->ptp_clock = NULL;
 949                 return;
 950         }
 951
 952         /* initialize the ptp filter */
 953         if (ptp_filter_init(ptp_filter, ARRAY_SIZE(ptp_filter)))
 954                 e_dev_warn("ptp_filter_init failed\n");
 955
 956         spin_lock_init(&adapter->tmreg_lock);
 957
 958         ixgbe_ptp_start_cyclecounter(adapter);
 959
 960         /* (Re)start the overflow check */
 961         adapter->flags2 |= IXGBE_FLAG2_OVERFLOW_CHECK_ENABLED;
 962
 963         adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps);
 964         if (IS_ERR(adapter->ptp_clock)) {
 965                 adapter->ptp_clock = NULL;
 966                 e_dev_err("ptp_clock_register failed\n");
 967         } else
 968                 e_dev_info("registered PHC device on %s\n", netdev->name);
 969
 970         return;
 971 }
 972
 973 /**
 974  * ixgbe_ptp_stop - disable ptp device and stop the overflow check
 975  * @adapter: pointer to adapter struct
 976  *
 977  * this function stops the ptp support, and cancels the delayed work.
 978  */
 979 void ixgbe_ptp_stop(struct ixgbe_adapter *adapter)
 980 {
 981         ixgbe_ptp_disable_sdp(&adapter->hw);
 982
 983         /* stop the overflow check task */
 984         adapter->flags2 &= ~IXGBE_FLAG2_OVERFLOW_CHECK_ENABLED;
 985
 986         if (adapter->ptp_clock) {
 987                 ptp_clock_unregister(adapter->ptp_clock);
 988                 adapter->ptp_clock = NULL;
 989                 e_dev_info("removed PHC on %s\n",
 990                            adapter->netdev->name);
 991         }
 992 }